fix(name): moved copy object to upper level of the function

- updated markdown files
- updated dockerfiles
- updated test files
- updated webpack

.github/ISSUE_TEMPLATE.md

@@ -28,17 +28,15 @@ Briefly describe the problem you are having in a few paragraphs.
 
 **Steps to reproduce the issue:**
 
-1.
-2.
-3.
+1. 2. 3.
 
 Any other info e.g. Why do you consider this to be a bug? What did you expect to happen instead?
 
 **Technical details:**
 
-* Portainer version:
-* Target Docker version (the host/cluster you manage):
-* Platform (windows/linux):
-* Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
-* Target Swarm version (if applicable):
-* Browser:
+- Portainer version:
+- Target Docker version (the host/cluster you manage):
+- Platform (windows/linux):
+- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
+- Target Swarm version (if applicable):
+- Browser:

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -4,7 +4,6 @@ about: Create a bug report
 title: ''
 labels: bug/need-confirmation, kind/bug
 assignees: ''
-
 ---
 
 <!--
@@ -46,7 +45,7 @@ You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#ho
 - Docker version (managed by Portainer):
 - Kubernetes version (managed by Portainer):
 - Platform (windows/linux):
-- Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
+- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
 - Browser:
 - Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
 - Have you reviewed our technical documentation and knowledge base? Yes/No

.github/stale.yml

@@ -1,54 +0,0 @@
-# Config for Stalebot, limited to only `issues`
-only: issues
-
-# Issues config
-issues:
-  daysUntilStale: 60
-  daysUntilClose: 7
-
-  # Limit the number of actions per hour, from 1-30. Default is 30
-  limitPerRun: 30
-
-  # Issues with these labels will never be considered stale
-  exemptLabels:
-    - kind/enhancement
-    - kind/question
-    - kind/style
-    - kind/workaround
-    - kind/refactor
-    - bug/need-confirmation
-    - bug/confirmed
-    - status/discuss
-
-  # Only issues with all of these labels are checked if stale. Defaults to `[]` (disabled)
-  onlyLabels: []
-
-  # Set to true to ignore issues in a project (defaults to false)
-  exemptProjects: true
-  # Set to true to ignore issues in a milestone (defaults to false)
-  exemptMilestones: true
-  # Set to true to ignore issues with an assignee (defaults to false)
-  exemptAssignees: true
-
-  # Label to use when marking an issue as stale
-  staleLabel: status/stale
-
-  # Comment to post when marking an issue as stale. Set to `false` to disable
-  markComment: >
-    This issue has been marked as stale as it has not had recent activity,
-    it will be closed if no further activity occurs in the next 7 days.
-    If you believe that it has been incorrectly labelled as stale,
-    leave a comment and the label will be removed.
-
-  # Comment to post when removing the stale label.
-  # unmarkComment: >
-  #   Your comment here.
-
-  # Comment to post when closing a stale issue. Set to `false` to disable
-  closeComment: >
-    Since no further activity has appeared on this issue it will be closed.
-    If you believe that it has been incorrectly closed, leave a comment
-    mentioning `ametdoohan`, `balasu` or `keverv` and one of our staff will then review the issue.
-
-    Note - If it is an old bug report, make sure that it is reproduceable in the
-    latest version of Portainer as it may have already been fixed.

.github/workflows/label-conflcts.yaml

@@ -0,0 +1,15 @@
+on:
+  push:
+    branches:
+      - develop
+      - 'release/**'
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mschilde/auto-label-merge-conflicts@master
+        with:
+          CONFLICT_LABEL_NAME: 'has conflicts'
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MAX_RETRIES: 5
+          WAIT_MS: 5000

.github/workflows/stale.yml

@@ -0,0 +1,27 @@
+name: Close Stale Issues
+on:
+  schedule:
+  - cron: '0 12 * * *'
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+
+    steps:
+    - uses: actions/stale@v4.0.0
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+        # Issue Config
+        days-before-issue-stale: 60
+        days-before-issue-close: 7
+        stale-issue-label: 'status/stale'
+        exempt-all-issue-milestones: true # Do not stale issues in a milestone
+        exempt-issue-labels: kind/enhancement, kind/style, kind/workaround, kind/refactor, bug/need-confirmation, bug/confirmed, status/discuss
+        stale-issue-message: 'This issue has been marked as stale as it has not had recent activity, it will be closed if no further activity occurs in the next 7 days. If you believe that it has been incorrectly labelled as stale, leave a comment and the label will be removed.'
+        close-issue-message: 'Since no further activity has appeared on this issue it will be closed. If you believe that it has been incorrectly closed, leave a comment mentioning `portainer/support` and one of our staff will then review the issue. Note - If it is an old bug report, make sure that it is reproduceable in the latest version of Portainer as it may have already been fixed.'
+        
+        # Pull Request Config
+        days-before-pr-stale: -1 # Do not stale pull request
+        days-before-pr-close: -1 # Do not close pull request

.vscode.example/portainer.code-snippets

@@ -163,5 +163,19 @@
       "// @failure 500 \"Server error\"",
       "// @router /{id} [get]"
     ]
+  },
+  "analytics": {
+    "prefix": "nlt",
+    "body": ["analytics-on", "analytics-category=\"$1\"", "analytics-event=\"$2\""],
+    "description": "analytics"
+  },
+  "analytics-if": {
+    "prefix": "nltf",
+    "body": ["analytics-if=\"$1\""],
+    "description": "analytics"
+  },
+  "analytics-metadata": {
+    "prefix": "nltm",
+    "body": "analytics-properties=\"{ metadata: { $1 } }\""
   }
 }

.vscode.example/settings.json

@@ -0,0 +1,4 @@
+{
+  "go.lintTool": "golangci-lint",
+  "go.lintFlags": ["--fast", "-E", "exportloopref"]
+}

CONTRIBUTING.md

@@ -91,7 +91,7 @@ Then build and run the project:
 $ yarn start
 ```
 
-Portainer can now be accessed at <http://localhost:9000>.
+Portainer can now be accessed at <https://localhost:9443>.
 
 Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
 

README.md

@@ -1,16 +1,14 @@
 <p align="center">
-  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/logo_alt.png?raw=true' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/portainer-github-banner.png?raw=true' />
 </p>
 
-[![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
-[![Microbadger](https://images.microbadger.com/badges/image/portainer/portainer.svg)](http://microbadger.com/images/portainer/portainer 'Image size')
-[![Build Status](https://portainer.visualstudio.com/Portainer%20CI/_apis/build/status/Portainer%20CI?branchName=develop)](https://portainer.visualstudio.com/Portainer%20CI/_build/latest?definitionId=3&branchName=develop)
-[![Code Climate](https://codeclimate.com/github/portainer/portainer/badges/gpa.svg)](https://codeclimate.com/github/portainer/portainer)
-[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YHXZJQNJQ36H6)
+**Portainer CE** is a lightweight ‘universal’ management GUI that can be used to **easily** manage Docker, Swarm, Kubernetes and ACI environments. It is designed to be as **simple** to deploy as it is to use.
 
-**_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
-**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container, supports other platforms too).
-**_Portainer_** allows you to manage all your Docker resources (containers, images, volumes, networks and more!) It is compatible with the _standalone Docker_ engine and with _Docker Swarm mode_.
+Portainer consists of a single container that can run on any cluster. It can be deployed as a Linux container or a Windows native container.
+
+**Portainer** allows you to manage all your orchestrator resources (containers, images, volumes, networks and more) through a super-simple graphical interface.
+
+A fully supported version of Portainer is available for business use. Visit http://www.portainer.io to learn more
 
 ## Demo
 
@@ -18,30 +16,38 @@ You can try out the public demo instance: http://demo.portainer.io/ (login with
 
 Please note that the public demo cluster is **reset every 15min**.
 
-Alternatively, you can deploy a copy of the demo stack inside a [play-with-docker (PWD)](https://labs.play-with-docker.com) playground:
+## Latest Version
 
-- Browse [PWD/?stack=portainer-demo/play-with-docker/docker-stack.yml](http://play-with-docker.com/?stack=https://raw.githubusercontent.com/portainer/portainer-demo/master/play-with-docker/docker-stack.yml)
-- Sign in with your [Docker ID](https://docs.docker.com/docker-id)
-- Follow [these](https://github.com/portainer/portainer-demo/blob/master/play-with-docker/docker-stack.yml#L5-L8) steps.
+Portainer CE is updated regularly. We aim to do an update release every couple of months.
 
-Unlike the public demo, the playground sessions are deleted after 4 hours. Apart from that, all the settings are the same, including default credentials.
+**The latest version of Portainer is 2.6.x** And you can find the release notes [here.](https://www.portainer.io/blog/new-portainer-ce-2.6.0-release)
+Portainer is on version 2, the second number denotes the month of release.
 
 ## Getting started
 
 - [Deploy Portainer](https://documentation.portainer.io/quickstart/)
 - [Documentation](https://documentation.portainer.io)
-- [Building Portainer](https://documentation.portainer.io/contributing/instructions/)
+- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
+
+## Features & Functions
+
+View [this](https://www.portainer.io/products) table to see all of the Portainer CE functionality and compare to Portainer Business.
+
+- [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
+- [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
+- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
 
 ## Getting help
 
-For FORMAL Support, please purchase a support subscription from here: https://www.portainer.io/products/portainer-business
+Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-For community support: You can find more information about Portainer's community support framework policy here: https://www.portainer.io/products/community-edition/customer-success
+Learn more about Portainers community support channels [here.](https://www.portainer.io/help_about)
 
 - Issues: https://github.com/portainer/portainer/issues
-- FAQ: https://documentation.portainer.io
 - Slack (chat): https://portainer.io/slack/
 
+You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
+
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
@@ -51,6 +57,10 @@ For community support: You can find more information about Portainer's community
 
 - Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
 
+## WORK FOR US
+
+If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and we will be in touch. 
+
 ## Privacy
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**

api/api-description.md

@@ -1,5 +1,5 @@
 Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
-Examples are available at https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8
+Examples are available at https://documentation.portainer.io/api/api-examples/
 You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
 
 # Authentication
@@ -50,4 +50,4 @@ Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you
 
 To do so, you can use the `/endpoints/{id}/docker` Portainer API endpoint (which is not documented below due to Swagger limitations). This endpoint has a restricted access policy so you still need to be authenticated to be able to query this endpoint. Any query on this endpoint will be proxied to the Docker API of the associated endpoint (requests and responses objects are the same as documented in the Docker API).
 
-**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8).
+**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).

api/backup/backup.go

@@ -10,12 +10,24 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
 const rwxr__r__ os.FileMode = 0744
 
-var filesToBackup = []string{"compose", "config.json", "custom_templates", "edge_jobs", "edge_stacks", "extensions", "portainer.key", "portainer.pub", "tls"}
+var filesToBackup = []string{
+	"certs",
+	"compose",
+	"config.json",
+	"custom_templates",
+	"edge_jobs",
+	"edge_stacks",
+	"extensions",
+	"portainer.key",
+	"portainer.pub",
+	"tls",
+}
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
 func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
@@ -32,7 +44,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 	}
 
 	for _, filename := range filesToBackup {
-		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
+		err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath)
 		if err != nil {
 			return "", errors.Wrap(err, "Failed to create backup file")
 		}

api/backup/copy_test.go

@@ -1,105 +0,0 @@
-package backup
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func contains(t *testing.T, list []string, path string) {
-	assert.Contains(t, list, path)
-	copyContent, _ := ioutil.ReadFile(path)
-	assert.Equal(t, "content\n", string(copyContent))
-}
-
-func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyFile("does-not-exist", tmpdir)
-	assert.NotNil(t, err)
-}
-
-func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
-
-	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
-	assert.Nil(t, err)
-
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyDir("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}
-
-func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyPath("does-not-exists", tmpdir)
-	assert.Nil(t, err)
-
-	assert.Empty(t, listFiles(tmpdir))
-}
-
-func Test_backupPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
-
-	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
-	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
-	assert.Nil(t, err)
-
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
-	assert.Nil(t, err)
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_backupPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyPath("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}

api/backup/restore.go

@@ -11,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
@@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error {
 
 func restoreFiles(srcDir string, destinationDir string) error {
 	for _, filename := range filesToRestore {
-		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
+		err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)
 		if err != nil {
 			return err
 		}

api/bolt/bolttest/datastore.go

@@ -0,0 +1,73 @@
+package bolttest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+var errTempDir = errors.New("can't create a temp dir")
+
+func MustNewTestStore(init bool) (*bolt.Store, func()) {
+	store, teardown, err := NewTestStore(init)
+	if err != nil {
+		if !errors.Is(err, errTempDir) {
+			teardown()
+		}
+		log.Fatal(err)
+	}
+
+	return store, teardown
+}
+
+func NewTestStore(init bool) (*bolt.Store, func(), error) {
+	// Creates unique temp directory in a concurrency friendly manner.
+	dataStorePath, err := ioutil.TempDir("", "boltdb")
+	if err != nil {
+		return nil, nil, errors.Wrap(errTempDir, err.Error())
+	}
+
+	fileService, err := filesystem.NewService(dataStorePath, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	store, err := bolt.NewStore(dataStorePath, fileService)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = store.Open()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if init {
+		err = store.Init()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	teardown := func() {
+		teardown(store, dataStorePath)
+	}
+
+	return store, teardown, nil
+}
+
+func teardown(store *bolt.Store, dataStorePath string) {
+	err := store.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	err = os.RemoveAll(dataStorePath)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

api/bolt/datastore.go

@@ -6,6 +6,8 @@ import (
 	"path"
 	"time"
 
+	"github.com/portainer/portainer/api/bolt/helmuserrepository"
+
 	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/customtemplate"
@@ -25,6 +27,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/role"
 	"github.com/portainer/portainer/api/bolt/schedule"
 	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
@@ -43,32 +46,34 @@ const (
 // Store defines the implementation of portainer.DataStore using
 // BoltDB as the storage system.
 type Store struct {
-	path                    string
-	connection              *internal.DbConnection
-	isNew                   bool
-	fileService             portainer.FileService
-	CustomTemplateService   *customtemplate.Service
-	DockerHubService        *dockerhub.Service
-	EdgeGroupService        *edgegroup.Service
-	EdgeJobService          *edgejob.Service
-	EdgeStackService        *edgestack.Service
-	EndpointGroupService    *endpointgroup.Service
-	EndpointService         *endpoint.Service
-	EndpointRelationService *endpointrelation.Service
-	ExtensionService        *extension.Service
-	RegistryService         *registry.Service
-	ResourceControlService  *resourcecontrol.Service
-	RoleService             *role.Service
-	ScheduleService         *schedule.Service
-	SettingsService         *settings.Service
-	StackService            *stack.Service
-	TagService              *tag.Service
-	TeamMembershipService   *teammembership.Service
-	TeamService             *team.Service
-	TunnelServerService     *tunnelserver.Service
-	UserService             *user.Service
-	VersionService          *version.Service
-	WebhookService          *webhook.Service
+	path                      string
+	connection                *internal.DbConnection
+	isNew                     bool
+	fileService               portainer.FileService
+	CustomTemplateService     *customtemplate.Service
+	DockerHubService          *dockerhub.Service
+	EdgeGroupService          *edgegroup.Service
+	EdgeJobService            *edgejob.Service
+	EdgeStackService          *edgestack.Service
+	EndpointGroupService      *endpointgroup.Service
+	EndpointService           *endpoint.Service
+	EndpointRelationService   *endpointrelation.Service
+	ExtensionService          *extension.Service
+	HelmUserRepositoryService *helmuserrepository.Service
+	RegistryService           *registry.Service
+	ResourceControlService    *resourcecontrol.Service
+	RoleService               *role.Service
+	ScheduleService           *schedule.Service
+	SettingsService           *settings.Service
+	SSLSettingsService        *ssl.Service
+	StackService              *stack.Service
+	TagService                *tag.Service
+	TeamMembershipService     *teammembership.Service
+	TeamService               *team.Service
+	TunnelServerService       *tunnelserver.Service
+	UserService               *user.Service
+	VersionService            *version.Service
+	WebhookService            *webhook.Service
 }
 
 func (store *Store) edition() portainer.SoftwareEdition {
@@ -114,6 +119,7 @@ func (store *Store) Open() error {
 }
 
 // Close closes the BoltDB database.
+// Safe to being called multiple times.
 func (store *Store) Close() error {
 	if store.connection.DB != nil {
 		return store.connection.Close()
@@ -169,6 +175,7 @@ func (store *Store) MigrateData(force bool) error {
 			UserService:             store.UserService,
 			VersionService:          store.VersionService,
 			FileService:             store.fileService,
+			DockerhubService:        store.DockerHubService,
 			AuthorizationService:    authorization.NewService(store),
 		}
 		migrator := migrator.NewMigrator(migratorParams)

api/bolt/errors/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 var (
 	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documention to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 )

api/bolt/helmuserrepository/helmuserrepository.go

@@ -0,0 +1,73 @@
+package helmuserrepository
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "helm_user_repository"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
+func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
+	var result = make([]portainer.HelmUserRepository, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var record portainer.HelmUserRepository
+			err := internal.UnmarshalObject(v, &record)
+			if err != nil {
+				return err
+			}
+
+			if record.UserID == userID {
+				result = append(result, record)
+			}
+		}
+
+		return nil
+	})
+
+	return result, err
+}
+
+// CreateHelmUserRepository creates a new HelmUserRepository object.
+func (service *Service) CreateHelmUserRepository(record *portainer.HelmUserRepository) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		record.ID = portainer.HelmUserRepositoryID(id)
+
+		data, err := internal.MarshalObject(record)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(record.ID)), data)
+	})
+}

api/bolt/init.go

@@ -44,7 +44,9 @@ func (store *Store) Init() error {
 
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
 			TemplatesURL:             portainer.DefaultTemplatesURL,
+			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
+			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)
@@ -55,20 +57,20 @@ func (store *Store) Init() error {
 		return err
 	}
 
-	_, err = store.DockerHubService.DockerHub()
-	if err == errors.ErrObjectNotFound {
-		defaultDockerHub := &portainer.DockerHub{
-			Authentication: false,
-			Username:       "",
-			Password:       "",
+	_, err = store.SSLSettings().Settings()
+	if err != nil {
+		if err != errors.ErrObjectNotFound {
+			return err
 		}
 
-		err := store.DockerHubService.UpdateDockerHub(defaultDockerHub)
+		defaultSSLSettings := &portainer.SSLSettings{
+			HTTPEnabled: true,
+		}
+
+		err = store.SSLSettings().UpdateSettings(defaultSSLSettings)
 		if err != nil {
 			return err
 		}
-	} else if err != nil {
-		return err
 	}
 
 	groups, err := store.EndpointGroupService.EndpointGroups()
@@ -79,7 +81,7 @@ func (store *Store) Init() error {
 	if len(groups) == 0 {
 		unassignedGroup := &portainer.EndpointGroup{
 			Name:               "Unassigned",
-			Description:        "Unassigned endpoints",
+			Description:        "Unassigned environments",
 			Labels:             []portainer.Pair{},
 			UserAccessPolicies: portainer.UserAccessPolicies{},
 			TeamAccessPolicies: portainer.TeamAccessPolicies{},

api/bolt/log/log.go

@@ -0,0 +1,41 @@
+package log
+
+import (
+	"fmt"
+	"log"
+)
+
+const (
+	INFO  = "INFO"
+	ERROR = "ERROR"
+	DEBUG = "DEBUG"
+	FATAL = "FATAL"
+)
+
+type ScopedLog struct {
+	scope string
+}
+
+func NewScopedLog(scope string) *ScopedLog {
+	return &ScopedLog{scope: scope}
+}
+
+func (slog *ScopedLog) print(kind string, message string) {
+	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
+}
+
+func (slog *ScopedLog) Debug(message string) {
+	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Info(message string) {
+	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Error(message string, err error) {
+	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
+}
+
+func (slog *ScopedLog) NotImplemented(method string) {
+	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
+}

api/bolt/log/log.test.go

@@ -0,0 +1 @@
+package log

api/bolt/migrator/migrate_dbversion29.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) migrateDBVersionToDB30() error {
+	if err := m.migrateSettingsToDB30(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) migrateSettingsToDB30() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	legacySettings.OAuthSettings.SSO = false
+	legacySettings.OAuthSettings.LogoutURI = ""
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion29_test.go

@@ -0,0 +1,95 @@
+package migrator
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+var (
+	testingDBStorePath string
+	testingDBFileName  string
+	dummyLogoURL       string
+	dbConn             *bolt.DB
+	settingsService    *settings.Service
+)
+
+// initTestingDBConn creates a raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
+	databasePath := path.Join(storePath, fileName)
+	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+	return dbConn, nil
+}
+
+// initTestingDBConn creates a settings service with raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
+	internalDBConn := &internal.DbConnection{
+		DB: dbConn,
+	}
+	settingsService, err := settings.NewService(internalDBConn)
+	if err != nil {
+		return nil, err
+	}
+	//insert a obj
+	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
+		return nil, err
+	}
+	return settingsService, nil
+}
+
+func setup() error {
+	testingDBStorePath, _ = os.Getwd()
+	testingDBFileName = "portainer-ee-mig-30.db"
+	dummyLogoURL = "example.com"
+	var err error
+	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
+	if err != nil {
+		return err
+	}
+	dummySettingsObj := map[string]interface{}{
+		"LogoURL": dummyLogoURL,
+	}
+	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestMigrateSettings(t *testing.T) {
+	if err := setup(); err != nil {
+		t.Errorf("failed to complete testing setups, err: %v", err)
+	}
+	defer dbConn.Close()
+	defer os.Remove(testingDBFileName)
+	m := &Migrator{
+		db:              dbConn,
+		settingsService: settingsService,
+	}
+	if err := m.migrateSettingsToDB30(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+	updatedSettings, err := m.settingsService.Settings()
+	if err != nil {
+		t.Errorf("failed to retrieve the updated settings: %v", err)
+	}
+	if updatedSettings.LogoURL != dummyLogoURL {
+		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
+	}
+	if updatedSettings.OAuthSettings.SSO != false {
+		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
+	}
+	if updatedSettings.OAuthSettings.LogoutURI != "" {
+		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
+	}
+}

api/bolt/migrator/migrate_dbversion31.go

@@ -0,0 +1,239 @@
+package migrator
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
+)
+
+func (m *Migrator) migrateDBVersionToDB32() error {
+	err := m.updateRegistriesToDB32()
+	if err != nil {
+		return err
+	}
+
+	err = m.updateDockerhubToDB32()
+	if err != nil {
+		return err
+	}
+
+	if err := m.updateVolumeResourceControlToDB32(); err != nil {
+		return err
+	}
+
+	if err := m.kubeconfigExpiryToDB32(); err != nil {
+		return err
+	}
+
+	if err := m.helmRepositoryURLToDB32(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateRegistriesToDB32() error {
+	registries, err := m.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range registries {
+
+		registry.RegistryAccesses = portainer.RegistryAccesses{}
+
+		for _, endpoint := range endpoints {
+
+			filteredUserAccessPolicies := portainer.UserAccessPolicies{}
+			for userId, registryPolicy := range registry.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					filteredUserAccessPolicies[userId] = registryPolicy
+				}
+			}
+
+			filteredTeamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId, registryPolicy := range registry.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					filteredTeamAccessPolicies[teamId] = registryPolicy
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: filteredUserAccessPolicies,
+				TeamAccessPolicies: filteredTeamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+		m.registryService.UpdateRegistry(registry.ID, &registry)
+	}
+	return nil
+}
+
+func (m *Migrator) updateDockerhubToDB32() error {
+	dockerhub, err := m.dockerhubService.DockerHub()
+	if err == errors.ErrObjectNotFound {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if !dockerhub.Authentication {
+		return nil
+	}
+
+	registry := &portainer.Registry{
+		Type:             portainer.DockerHubRegistry,
+		Name:             "Dockerhub (authenticated - migrated)",
+		URL:              "docker.io",
+		Authentication:   true,
+		Username:         dockerhub.Username,
+		Password:         dockerhub.Password,
+		RegistryAccesses: portainer.RegistryAccesses{},
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+
+		if endpoint.Type != portainer.KubernetesLocalEnvironment &&
+			endpoint.Type != portainer.AgentOnKubernetesEnvironment &&
+			endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
+
+			userAccessPolicies := portainer.UserAccessPolicies{}
+			for userId := range endpoint.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					userAccessPolicies[userId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			teamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId := range endpoint.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					teamAccessPolicies[teamId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: userAccessPolicies,
+				TeamAccessPolicies: teamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+	}
+
+	return m.registryService.CreateRegistry(registry)
+}
+
+func (m *Migrator) updateVolumeResourceControlToDB32() error {
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return fmt.Errorf("failed fetching environments: %w", err)
+	}
+
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return fmt.Errorf("failed fetching resource controls: %w", err)
+	}
+
+	toUpdate := map[portainer.ResourceControlID]string{}
+	volumeResourceControls := map[string]*portainer.ResourceControl{}
+
+	for i := range resourceControls {
+		resourceControl := resourceControls[i]
+		if resourceControl.Type == portainer.VolumeResourceControl {
+			volumeResourceControls[resourceControl.ResourceID] = &resourceControl
+		}
+	}
+
+	for _, endpoint := range endpoints {
+		if !endpointutils.IsDockerEndpoint(&endpoint) {
+			continue
+		}
+
+		totalSnapshots := len(endpoint.Snapshots)
+		if totalSnapshots == 0 {
+			continue
+		}
+
+		snapshot := endpoint.Snapshots[totalSnapshots-1]
+
+		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
+		if err != nil {
+			return fmt.Errorf("failed fetching environment docker id: %w", err)
+		}
+
+		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
+			if volumesData["Volumes"] == nil {
+				continue
+			}
+
+			findResourcesToUpdateForDB32(endpointDockerID, volumesData, toUpdate, volumeResourceControls)
+		}
+	}
+
+	for _, resourceControl := range volumeResourceControls {
+		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
+			resourceControl.ResourceID = newResourceID
+			err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
+			if err != nil {
+				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
+			}
+
+		} else {
+			err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
+			}
+
+		}
+	}
+
+	return nil
+}
+
+func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interface{}, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
+	volumes := volumesData["Volumes"].([]interface{})
+	for _, volumeMeta := range volumes {
+		volume := volumeMeta.(map[string]interface{})
+		volumeName := volume["Name"].(string)
+		oldResourceID := fmt.Sprintf("%s%s", volumeName, volume["CreatedAt"].(string))
+		resourceControl, ok := volumeResourceControls[oldResourceID]
+
+		if ok {
+			toUpdate[resourceControl.ID] = fmt.Sprintf("%s_%s", volumeName, dockerID)
+		}
+	}
+}
+
+func (m *Migrator) kubeconfigExpiryToDB32() error {
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	settings.KubeconfigExpiry = portainer.DefaultKubeconfigExpiry
+	return m.settingsService.UpdateSettings(settings)
+}
+
+func (m *Migrator) helmRepositoryURLToDB32() error {
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	settings.HelmRepositoryURL = portainer.DefaultHelmRepositoryURL
+	return m.settingsService.UpdateSettings(settings)
+}

api/bolt/migrator/migrate_dbversion33.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) migrateDBVersionTo33() error {
+	err := migrateStackEntryPoint(m.stackService)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func migrateStackEntryPoint(stackService portainer.StackService) error {
+	stacks, err := stackService.Stacks()
+	if err != nil {
+		return err
+	}
+	for i := range stacks {
+		stack := &stacks[i]
+		if stack.GitConfig == nil {
+			continue
+		}
+		stack.GitConfig.ConfigFilePath = stack.EntryPoint
+		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/bolt/migrator/migrate_dbversion33_test.go

@@ -0,0 +1,51 @@
+package migrator
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateStackEntryPoint(t *testing.T) {
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	assert.NoError(t, err, "failed to init testing DB connection")
+	defer dbConn.Close()
+
+	stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn})
+	assert.NoError(t, err, "failed to init testing Stack service")
+
+	stacks := []*portainer.Stack{
+		{
+			ID:         1,
+			EntryPoint: "dir/sub/compose.yml",
+		},
+		{
+			ID:         2,
+			EntryPoint: "dir/sub/compose.yml",
+			GitConfig:  &gittypes.RepoConfig{},
+		},
+	}
+
+	for _, s := range stacks {
+		err := stackService.CreateStack(s)
+		assert.NoError(t, err, "failed to create stack")
+	}
+
+	err = migrateStackEntryPoint(stackService)
+	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
+
+	s, err := stackService.Stack(1)
+	assert.NoError(t, err)
+	assert.Nil(t, s.GitConfig, "first stack should not have git config")
+
+	s, err = stackService.Stack(2)
+	assert.NoError(t, err)
+	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")
+}

api/bolt/migrator/migrator.go

@@ -3,10 +3,12 @@ package migrator
 import (
 	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
+	plog "github.com/portainer/portainer/api/bolt/log"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -20,6 +22,8 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
+var migrateLog = plog.NewScopedLog("bolt, migrate")
+
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
@@ -41,6 +45,7 @@ type (
 		versionService          *version.Service
 		fileService             portainer.FileService
 		authorizationService    *authorization.Service
+		dockerhubService        *dockerhub.Service
 	}
 
 	// Parameters represents the required parameters to create a new Migrator instance.
@@ -63,6 +68,7 @@ type (
 		VersionService          *version.Service
 		FileService             portainer.FileService
 		AuthorizationService    *authorization.Service
+		DockerhubService        *dockerhub.Service
 	}
 )
 
@@ -87,6 +93,7 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		versionService:          parameters.VersionService,
 		fileService:             parameters.FileService,
 		authorizationService:    parameters.AuthorizationService,
+		dockerhubService:        parameters.DockerhubService,
 	}
 }
 
@@ -358,5 +365,27 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// Portainer 2.6.0
+	if m.currentDBVersion < 30 {
+		err := m.migrateDBVersionToDB30()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.9.0
+	if m.currentDBVersion < 32 {
+		err := m.migrateDBVersionToDB32()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 33 {
+		if err := m.migrateDBVersionTo33(); err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/services.go

@@ -11,11 +11,13 @@ import (
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/helmuserrepository"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
 	"github.com/portainer/portainer/api/bolt/schedule"
 	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
@@ -87,6 +89,12 @@ func (store *Store) initServices() error {
 	}
 	store.ExtensionService = extensionService
 
+	helmUserRepositoryService, err := helmuserrepository.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.HelmUserRepositoryService = helmUserRepositoryService
+
 	registryService, err := registry.NewService(store.connection)
 	if err != nil {
 		return err
@@ -105,6 +113,12 @@ func (store *Store) initServices() error {
 	}
 	store.SettingsService = settingsService
 
+	sslSettingsService, err := ssl.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SSLSettingsService = sslSettingsService
+
 	stackService, err := stack.NewService(store.connection)
 	if err != nil {
 		return err
@@ -167,11 +181,6 @@ func (store *Store) CustomTemplate() portainer.CustomTemplateService {
 	return store.CustomTemplateService
 }
 
-// DockerHub gives access to the DockerHub data management layer
-func (store *Store) DockerHub() portainer.DockerHubService {
-	return store.DockerHubService
-}
-
 // EdgeGroup gives access to the EdgeGroup data management layer
 func (store *Store) EdgeGroup() portainer.EdgeGroupService {
 	return store.EdgeGroupService
@@ -202,6 +211,11 @@ func (store *Store) EndpointRelation() portainer.EndpointRelationService {
 	return store.EndpointRelationService
 }
 
+// HelmUserRepository access the helm user repository settings
+func (store *Store) HelmUserRepository() portainer.HelmUserRepositoryService {
+	return store.HelmUserRepositoryService
+}
+
 // Registry gives access to the Registry data management layer
 func (store *Store) Registry() portainer.RegistryService {
 	return store.RegistryService
@@ -222,6 +236,11 @@ func (store *Store) Settings() portainer.SettingsService {
 	return store.SettingsService
 }
 
+// SSLSettings gives access to the SSL Settings data management layer
+func (store *Store) SSLSettings() portainer.SSLSettingsService {
+	return store.SSLSettingsService
+}
+
 // Stack gives access to the Stack data management layer
 func (store *Store) Stack() portainer.StackService {
 	return store.StackService

api/bolt/ssl/ssl.go

@@ -0,0 +1,46 @@
+package ssl
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "ssl"
+	key        = "SSL"
+)
+
+// Service represents a service for managing ssl data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Settings retrieve the ssl settings object.
+func (service *Service) Settings() (*portainer.SSLSettings, error) {
+	var settings portainer.SSLSettings
+
+	err := internal.GetObject(service.connection, BucketName, []byte(key), &settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return &settings, nil
+}
+
+// UpdateSettings persists a SSLSettings object.
+func (service *Service) UpdateSettings(settings *portainer.SSLSettings) error {
+	return internal.UpdateObject(service.connection, BucketName, []byte(key), settings)
+}

api/bolt/stack/stack.go

@@ -1,11 +1,14 @@
 package stack
 
 import (
+	"strings"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const (
@@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
 	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
+	if id == "" {
+		return nil, pkgerrors.New("webhook ID can't be empty string")
+	}
+	var stack portainer.Stack
+	found := false
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t struct {
+				AutoUpdate *struct {
+					WebhookID string `json:"Webhook"`
+				} `json:"AutoUpdate"`
+			}
+
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) {
+				found = true
+				err := internal.UnmarshalObject(v, &stack)
+				if err != nil {
+					return err
+				}
+				break
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.ErrObjectNotFound
+	}
+
+	return &stack, nil
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		var stack portainer.Stack
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+
+			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+				stacks = append(stacks, stack)
+			}
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}

api/bolt/stack/tests/stack_test.go

@@ -0,0 +1,111 @@
+package tests
+
+import (
+	"testing"
+	"time"
+
+	"github.com/portainer/portainer/api/bolt"
+
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+
+	"github.com/gofrs/uuid"
+
+	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+func newGuidString(t *testing.T) string {
+	uuid, err := uuid.NewV4()
+	assert.NoError(t, err)
+
+	return uuid.String()
+}
+
+type stackBuilder struct {
+	t     *testing.T
+	count int
+	store *bolt.Store
+}
+
+func TestService_StackByWebhookID(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	b := stackBuilder{t: t, store: store}
+	b.createNewStack(newGuidString(t))
+	for i := 0; i < 10; i++ {
+		b.createNewStack("")
+	}
+	webhookID := newGuidString(t)
+	stack := b.createNewStack(webhookID)
+
+	// can find a stack by webhook ID
+	got, err := store.StackService.StackByWebhookID(webhookID)
+	assert.NoError(t, err)
+	assert.Equal(t, stack, *got)
+
+	// returns nil and object not found error if there's no stack associated with the webhook
+	got, err = store.StackService.StackByWebhookID(newGuidString(t))
+	assert.Nil(t, got)
+	assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound)
+}
+
+func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
+	b.count++
+	stack := portainer.Stack{
+		ID:           portainer.StackID(b.count),
+		Name:         "Name",
+		Type:         portainer.DockerComposeStack,
+		EndpointID:   2,
+		EntryPoint:   filesystem.ComposeFileDefaultName,
+		Env:          []portainer.Pair{{"Name1", "Value1"}},
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+		ProjectPath:  "/tmp/project",
+		CreatedBy:    "test",
+	}
+
+	if webhookID == "" {
+		if b.count%2 == 0 {
+			stack.AutoUpdate = &portainer.StackAutoUpdate{
+				Interval: "",
+				Webhook:  "",
+			}
+		} // else keep AutoUpdate nil
+	} else {
+		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
+	}
+
+	err := b.store.StackService.CreateStack(&stack)
+	assert.NoError(b.t, err)
+
+	return stack
+}
+
+func Test_RefreshableStacks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	staticStack := portainer.Stack{ID: 1}
+	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
+	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
+
+	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
+		err := store.Stack().CreateStack(stack)
+		assert.NoError(t, err)
+	}
+
+	stacks, err := store.Stack().RefreshableStacks()
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
+}

api/chisel/service.go

@@ -141,7 +141,7 @@ func (service *Service) checkTunnels() {
 		}
 
 		elapsed := time.Since(tunnel.LastActivity)
-		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: endpoint tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
+		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
 
 		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
 			continue
@@ -156,19 +156,19 @@ func (service *Service) checkTunnels() {
 
 			endpointID, err := strconv.Atoi(item.Key)
 			if err != nil {
-				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
 			}
 
 			err = service.snapshotEnvironment(portainer.EndpointID(endpointID), tunnel.Port)
 			if err != nil {
-				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge endpoint (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %s): %s", item.Key, err)
 			}
 		}
 
 		if len(tunnel.Jobs) > 0 {
 			endpointID, err := strconv.Atoi(item.Key)
 			if err != nil {
-				log.Printf("[ERROR] [chisel,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [chisel,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
 				continue
 			}
 

api/cli/cli.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"time"
 
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -18,7 +18,7 @@ import (
 type Service struct{}
 
 var (
-	errInvalidEndpointProtocol       = errors.New("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errInvalidEndpointProtocol       = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
 	errSocketOrNamedPipeNotFound     = errors.New("Unable to locate Unix socket or named pipe")
 	errInvalidSnapshotInterval       = errors.New("Invalid snapshot interval")
 	errAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
@@ -30,11 +30,12 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 	flags := &portainer.CLIFlags{
 		Addr:                      kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
+		AddrHTTPS:                 kingpin.Flag("bind-https", "Address and port to serve Portainer via https").Default(defaultHTTPSBindAddress).String(),
 		TunnelAddr:                kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		EndpointURL:               kingpin.Flag("host", "Endpoint URL").Short('H').String(),
+		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
 		NoAnalytics:               kingpin.Flag("no-analytics", "Disable Analytics in app (deprecated)").Bool(),
 		TLS:                       kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
@@ -42,10 +43,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TLSCacert:                 kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:                   kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
 		TLSKey:                    kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
-		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
-		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
-		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
-		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
+		HTTPDisabled:              kingpin.Flag("http-disabled", "Serve portainer only on https").Default(defaultHTTPDisabled).Bool(),
+		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
+		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
+		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
+		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:                    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
@@ -92,6 +94,10 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 	if *flags.NoAnalytics {
 		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
 	}
+
+	if *flags.SSL {
+		log.Println("Warning: SSL is enabled by default and there is no need for the --ssl flag. It has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
+	}
 }
 
 func validateEndpointURL(endpointURL string) error {

api/cli/defaults.go

@@ -4,6 +4,7 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
+	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "/data"
@@ -13,6 +14,7 @@ const (
 	defaultTLSCACertPath       = "/certs/ca.pem"
 	defaultTLSCertPath         = "/certs/cert.pem"
 	defaultTLSKeyPath          = "/certs/key.pem"
+	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"

api/cli/defaults_windows.go

@@ -2,6 +2,7 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
+	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "C:\\data"
@@ -11,6 +12,7 @@ const (
 	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
 	defaultTLSCertPath         = "C:\\certs\\cert.pem"
 	defaultTLSKeyPath          = "C:\\certs\\key.pem"
+	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"

api/cmd/portainer/log.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"log"
+
+	"github.com/sirupsen/logrus"
+)
+
+func configureLogger() {
+	logger := logrus.New() // logger is to implicitly substitute stdlib's log
+	log.SetOutput(logger.Writer())
+
+	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
+	logger.SetFormatter(formatter)
+	logrus.SetFormatter(formatter)
+
+	logger.SetLevel(logrus.DebugLevel)
+	logrus.SetLevel(logrus.DebugLevel)
+}

api/cmd/portainer/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/docker"
 
+	"github.com/portainer/libhelm"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
@@ -20,14 +21,17 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
+	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
-	"github.com/portainer/portainer/api/libcompose"
 	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -52,7 +56,7 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
+func initDataStore(dataStorePath string, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
 	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
 		log.Fatalf("failed creating data store: %v", err)
@@ -72,24 +76,35 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 	if err != nil {
 		log.Fatalf("failed migration: %v", err)
 	}
+
+	go shutdownDatastore(shutdownCtx, store)
 	return store
 }
 
-func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper := exec.NewComposeWrapper(assetsPath, dataStorePath, proxyManager)
-	if composeWrapper != nil {
-		return composeWrapper
+func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStore) {
+	<-shutdownCtx.Done()
+	datastore.Close()
+}
+
+func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
+	if err != nil {
+		log.Fatalf("failed creating compose manager: %s", err)
 	}
 
-	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
+	return composeWrapper
 }
 
-func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
-	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
+func initSwarmStackManager(assetsPath string, configPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
+	return exec.NewSwarmStackManager(assetsPath, configPath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(assetsPath)
+func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, assetsPath)
+}
+
+func initHelmPackageManager(assetsPath string) (libhelm.HelmPackageManager, error) {
+	return libhelm.NewHelmPackageManager(libhelm.HelmConfig{BinaryPath: assetsPath})
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -102,7 +117,7 @@ func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error)
 		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
 		dataStore.Settings().UpdateSettings(settings)
 	}
-	jwtService, err := jwt.NewService(settings.UserSessionTimeout)
+	jwtService, err := jwt.NewService(settings.UserSessionTimeout, dataStore)
 	if err != nil {
 		return nil, err
 	}
@@ -129,12 +144,29 @@ func initGitService() portainer.GitService {
 	return git.NewService()
 }
 
+func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore portainer.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
+	slices := strings.Split(addr, ":")
+	host := slices[0]
+	if host == "" {
+		host = "0.0.0.0"
+	}
+
+	sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)
+
+	err := sslService.Init(host, certPath, keyPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return sslService, nil
+}
+
 func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore portainer.DataStore) *kubecli.ClientFactory {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
 }
 
 func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
@@ -149,9 +181,10 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
-func initStatus(flags *portainer.CLIFlags) *portainer.Status {
+func initStatus(instanceID string) *portainer.Status {
 	return &portainer.Status{
-		Version: portainer.APIVersion,
+		Version:    portainer.APIVersion,
+		InstanceID: instanceID,
 	}
 }
 
@@ -165,6 +198,7 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 	settings.SnapshotInterval = *flags.SnapshotInterval
 	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
 	settings.EnableTelemetry = true
+	settings.OAuthSettings.SSO = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -174,7 +208,26 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 		settings.BlackListedLabels = *flags.Labels
 	}
 
-	return dataStore.Settings().UpdateSettings(settings)
+	err = dataStore.Settings().UpdateSettings(settings)
+	if err != nil {
+		return err
+	}
+
+	httpEnabled := !*flags.HTTPDisabled
+
+	sslSettings, err := dataStore.SSLSettings().Settings()
+	if err != nil {
+		return err
+	}
+
+	sslSettings.HTTPEnabled = httpEnabled
+
+	err = dataStore.SSLSettings().UpdateSettings(sslSettings)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
@@ -240,6 +293,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -267,7 +321,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 	}
 
 	return dataStore.Endpoint().CreateEndpoint(endpoint)
@@ -301,6 +355,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -312,7 +367,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 	}
 
 	return dataStore.Endpoint().CreateEndpoint(endpoint)
@@ -329,7 +384,7 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	}
 
 	if len(endpoints) > 0 {
-		log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
+		log.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
 		return nil
 	}
 
@@ -344,7 +399,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, fileService)
+	dataStore := initDataStore(*flags.Data, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -365,6 +420,16 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	digitalSignatureService := initDigitalSignatureService()
 
+	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sslSettings, err := sslService.GetSSLSettings()
+	if err != nil {
+		log.Fatalf("failed to get ssl settings: %s", err)
+	}
+
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatalf("failed initializing key pai: %v", err)
@@ -378,7 +443,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
+	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
@@ -386,16 +451,30 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 	snapshotService.Start()
 
-	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
-	if err != nil {
-		log.Fatalf("failed initializing swarm stack manager: %v", err)
-	}
+	authorizationService := authorization.NewService(dataStore)
+	authorizationService.K8sClientFactory = kubernetesClientFactory
+
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
+
+	kubeConfigService := kubernetes.NewKubeConfigCAService(*flags.AddrHTTPS, sslSettings.CertPath)
+
 	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
 
-	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
+	dockerConfigPath := fileService.GetDockerConfigPath()
 
-	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
+	composeStackManager := initComposeStackManager(*flags.Assets, dockerConfigPath, reverseTunnelService, proxyManager)
+
+	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService)
+	if err != nil {
+		log.Fatalf("failed initializing swarm stack manager: %s", err)
+	}
+
+	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
+
+	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
+	if err != nil {
+		log.Fatalf("failed initializing helm package manager: %s", err)
+	}
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
@@ -409,11 +488,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed loading edge jobs from database: %v", err)
 	}
 
-	applicationStatus := initStatus(flags)
+	applicationStatus := initStatus(instanceID)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatalf("failed initializing endpoint: %v", err)
+		log.Fatalf("failed initializing environment: %v", err)
 	}
 
 	adminPasswordHash := ""
@@ -454,18 +533,31 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting license service: %s", err)
+		log.Fatalf("failed starting tunnel server: %s", err)
 	}
 
+	sslDBSettings, err := dataStore.SSLSettings().Settings()
+	if err != nil {
+		log.Fatalf("failed to fetch ssl settings from DB")
+	}
+
+	scheduler := scheduler.NewScheduler(shutdownCtx)
+	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager)
+	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
+
 	return &http.Server{
+		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
 		Status:                      applicationStatus,
 		BindAddress:                 *flags.Addr,
+		BindAddressHTTPS:            *flags.AddrHTTPS,
+		HTTPEnabled:                 sslDBSettings.HTTPEnabled,
 		AssetsPath:                  *flags.Assets,
 		DataStore:                   dataStore,
 		SwarmStackManager:           swarmStackManager,
 		ComposeStackManager:         composeStackManager,
 		KubernetesDeployer:          kubernetesDeployer,
+		HelmPackageManager:          helmPackageManager,
 		CryptoService:               cryptoService,
 		JWTService:                  jwtService,
 		FileService:                 fileService,
@@ -474,25 +566,28 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		GitService:                  gitService,
 		ProxyManager:                proxyManager,
 		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
+		KubeConfigService:           kubeConfigService,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
-		SSL:                         *flags.SSL,
-		SSLCert:                     *flags.SSLCert,
-		SSLKey:                      *flags.SSLKey,
+		SSLService:                  sslService,
 		DockerClientFactory:         dockerClientFactory,
 		KubernetesClientFactory:     kubernetesClientFactory,
+		Scheduler:                   scheduler,
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
+		StackDeployer:               stackDeployer,
 	}
 }
 
 func main() {
 	flags := initCLI()
 
+	configureLogger()
+
 	for {
 		server := buildServer(flags)
-		log.Printf("Starting Portainer %s on %s\n", portainer.APIVersion, *flags.Addr)
+		log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
 		err := server.Start()
-		log.Printf("Http server exited: %s\n", err)
+		log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
 	}
 }

api/docker/client.go

@@ -42,7 +42,7 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
 	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
 		return createAgentClient(endpoint, factory.signatureService, nodeName)
 	} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
+		return createEdgeClient(endpoint, factory.signatureService, factory.reverseTunnelService, nodeName)
 	}
 
 	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
@@ -71,13 +71,22 @@ func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	)
 }
 
-func createEdgeClient(endpoint *portainer.Endpoint, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
+func createEdgeClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
 	httpCli, err := httpClient(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	headers := map[string]string{}
+	signature, err := signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{
+		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
+		portainer.PortainerAgentSignatureHeader: signature,
+	}
+
 	if nodeName != "" {
 		headers[portainer.PortainerAgentTargetHeader] = nodeName
 	}

api/docker/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 // Docker errors
 var (
-	ErrUnableToPingEndpoint = errors.New("Unable to communicate with the endpoint")
+	ErrUnableToPingEndpoint = errors.New("Unable to communicate with the environment")
 )

api/docker/snapshot.go

@@ -47,44 +47,44 @@ func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Dock
 
 	err = snapshotInfo(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	if snapshot.Swarm {
 		err = snapshotSwarmServices(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [endpoint: %s] [err: %s]", endpoint.Name, err)
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [environment: %s] [err: %s]", endpoint.Name, err)
 		}
 
 		err = snapshotNodes(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [endpoint: %s] [err: %s]", endpoint.Name, err)
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [environment: %s] [err: %s]", endpoint.Name, err)
 		}
 	}
 
 	err = snapshotContainers(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotImages(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotVolumes(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotNetworks(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotVersion(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [environment: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	snapshot.Time = time.Now().Unix()

api/exec/compose_stack.go

@@ -0,0 +1,143 @@
+package exec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	libstack "github.com/portainer/docker-compose-wrapper"
+	"github.com/portainer/docker-compose-wrapper/compose"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/proxy/factory"
+)
+
+// ComposeStackManager is a wrapper for docker-compose binary
+type ComposeStackManager struct {
+	deployer     libstack.Deployer
+	proxyManager *proxy.Manager
+}
+
+// NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
+func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
+	deployer, err := compose.NewComposeDeployer(binaryPath, configPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ComposeStackManager{
+		deployer:     deployer,
+		proxyManager: proxyManager,
+	}, nil
+}
+
+// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
+func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
+	return portainer.ComposeSyntaxMaxVersion
+}
+
+// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
+func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	url, proxy, err := manager.fetchEndpointProxy(endpoint)
+	if err != nil {
+		return errors.Wrap(err, "failed to featch environment proxy")
+	}
+
+	if proxy != nil {
+		defer proxy.Close()
+	}
+
+	envFilePath, err := createEnvFile(stack)
+	if err != nil {
+		return errors.Wrap(err, "failed to create env file")
+	}
+
+	filePaths := getStackFiles(stack)
+	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFilePath)
+	return errors.Wrap(err, "failed to deploy a stack")
+}
+
+// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	url, proxy, err := manager.fetchEndpointProxy(endpoint)
+	if err != nil {
+		return err
+	}
+	if proxy != nil {
+		defer proxy.Close()
+	}
+
+	filePaths := getStackFiles(stack)
+	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths)
+	return errors.Wrap(err, "failed to remove a stack")
+}
+
+// NormalizeStackName returns a new stack name with unsupported characters replaced
+func (w *ComposeStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
+}
+
+func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
+	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
+		return "", nil, nil
+	}
+
+	proxy, err := manager.proxyManager.CreateComposeProxyServer(endpoint)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
+}
+
+func createEnvFile(stack *portainer.Stack) (string, error) {
+	if stack.Env == nil || len(stack.Env) == 0 {
+		return "", nil
+	}
+
+	envFilePath := path.Join(stack.ProjectPath, "stack.env")
+
+	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return "", err
+	}
+
+	for _, v := range stack.Env {
+		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
+	}
+	envfile.Close()
+
+	return "stack.env", nil
+}
+
+// getStackFiles returns list of stack's confile file paths.
+// items in the list would be sanitized according to following criterias:
+// 1. no empty paths
+// 2. no "../xxx" paths that are trying to escape stack folder
+// 3. no dir paths
+// 4. root paths would be made relative
+func getStackFiles(stack *portainer.Stack) []string {
+	paths := make([]string, 0, len(stack.AdditionalFiles)+1)
+
+	for _, p := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
+		if strings.HasPrefix(p, "/") {
+			p = `.` + p
+		}
+
+		if p == `` || p == `.` || strings.HasPrefix(p, `..`) || strings.HasSuffix(p, string(filepath.Separator)) {
+			continue
+		}
+
+		paths = append(paths, p)
+	}
+
+	return paths
+}

api/exec/compose_stack_integration_test.go

@@ -1,8 +1,7 @@
-// +build integration
-
 package exec
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -33,7 +32,9 @@ func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
 		Name:        "project-name",
 	}
 
-	endpoint := &portainer.Endpoint{}
+	endpoint := &portainer.Endpoint{
+		URL: "unix://",
+	}
 
 	return stack, endpoint
 }
@@ -42,18 +43,23 @@ func Test_UpAndDown(t *testing.T) {
 
 	stack, endpoint := setup(t)
 
-	w := NewComposeWrapper("", "", nil)
+	w, err := NewComposeStackManager("", "", nil)
+	if err != nil {
+		t.Fatalf("Failed creating manager: %s", err)
+	}
 
-	err := w.Up(stack, endpoint)
+	ctx := context.TODO()
+
+	err = w.Up(ctx, stack, endpoint)
 	if err != nil {
 		t.Fatalf("Error calling docker-compose up: %s", err)
 	}
 
-	if containerExists(composedContainerName) == false {
+	if !containerExists(composedContainerName) {
 		t.Fatal("container should exist")
 	}
 
-	err = w.Down(stack, endpoint)
+	err = w.Down(ctx, stack, endpoint)
 	if err != nil {
 		t.Fatalf("Error calling docker-compose down: %s", err)
 	}
@@ -63,13 +69,13 @@ func Test_UpAndDown(t *testing.T) {
 	}
 }
 
-func containerExists(contaierName string) bool {
-	cmd := exec.Command(osProgram("docker"), "ps", "-a", "-f", fmt.Sprintf("name=%s", contaierName))
+func containerExists(containerName string) bool {
+	cmd := exec.Command("docker", "ps", "-a", "-f", fmt.Sprintf("name=%s", containerName))
 
 	out, err := cmd.Output()
 	if err != nil {
 		log.Fatalf("failed to list containers: %s", err)
 	}
 
-	return strings.Contains(string(out), contaierName)
+	return strings.Contains(string(out), containerName)
 }

api/exec/compose_stack_test.go

@@ -0,0 +1,84 @@
+package exec
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_createEnvFile(t *testing.T) {
+	dir := t.TempDir()
+
+	tests := []struct {
+		name         string
+		stack        *portainer.Stack
+		expected     string
+		expectedFile bool
+	}{
+		{
+			name: "should not add env file option if stack doesn't have env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+			},
+			expected: "",
+		},
+		{
+			name: "should not add env file option if stack's env variables are empty",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env:         []portainer.Pair{},
+			},
+			expected: "",
+		},
+		{
+			name: "should add env file option if stack has env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env: []portainer.Pair{
+					{Name: "var1", Value: "value1"},
+					{Name: "var2", Value: "value2"},
+				},
+			},
+			expected: "var1=value1\nvar2=value2\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, _ := createEnvFile(tt.stack)
+
+			if tt.expected != "" {
+				assert.Equal(t, "stack.env", result)
+
+				f, _ := os.Open(path.Join(dir, "stack.env"))
+				content, _ := ioutil.ReadAll(f)
+
+				assert.Equal(t, tt.expected, string(content))
+			} else {
+				assert.Equal(t, "", result)
+			}
+		})
+	}
+}
+
+func Test_getStackFiles(t *testing.T) {
+	stack := &portainer.Stack{
+		EntryPoint: "./file", // picks entry point
+		AdditionalFiles: []string{
+			``,                  // ignores empty string
+			`.`,                 // ignores .
+			`..`,                // ignores ..
+			`./dir/`,            // ignrores paths that end with trailing /
+			`/with-root-prefix`, // replaces "root" based paths with relative
+			`./relative`,        // keeps relative paths
+			`../escape`,         // prevents dir escape
+		},
+	}
+
+	filePaths := getStackFiles(stack)
+	assert.ElementsMatch(t, filePaths, []string{`./file`, `./with-root-prefix`, `./relative`})
+}

api/exec/compose_wrapper.go

@@ -1,141 +0,0 @@
-package exec
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy"
-)
-
-// ComposeWrapper is a wrapper for docker-compose binary
-type ComposeWrapper struct {
-	binaryPath   string
-	dataPath     string
-	proxyManager *proxy.Manager
-}
-
-// NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeWrapper(binaryPath, dataPath string, proxyManager *proxy.Manager) *ComposeWrapper {
-	if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
-		return nil
-	}
-
-	return &ComposeWrapper{
-		binaryPath:   binaryPath,
-		dataPath:     dataPath,
-		proxyManager: proxyManager,
-	}
-}
-
-// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
-func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
-	return portainer.ComposeSyntaxMaxVersion
-}
-
-// NormalizeStackName returns a new stack name with unsupported characters replaced
-func (w *ComposeWrapper) NormalizeStackName(name string) string {
-	return name
-}
-
-// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
-func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	_, err := w.command([]string{"up", "-d"}, stack, endpoint)
-	return err
-}
-
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
-func (w *ComposeWrapper) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	_, err := w.command([]string{"down", "--remove-orphans"}, stack, endpoint)
-	return err
-}
-
-func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpoint *portainer.Endpoint) ([]byte, error) {
-	if endpoint == nil {
-		return nil, errors.New("cannot call a compose command on an empty endpoint")
-	}
-
-	program := programPath(w.binaryPath, "docker-compose")
-
-	options := setComposeFile(stack)
-
-	options = addProjectNameOption(options, stack)
-	options, err := addEnvFileOption(options, stack)
-	if err != nil {
-		return nil, err
-	}
-
-	if !(endpoint.URL == "" || strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://")) {
-
-		proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
-		if err != nil {
-			return nil, err
-		}
-
-		defer proxy.Close()
-
-		options = append(options, "-H", fmt.Sprintf("http://127.0.0.1:%d", proxy.Port))
-	}
-
-	args := append(options, command...)
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(program, args...)
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_CONFIG=%s", w.dataPath))
-	cmd.Stderr = &stderr
-
-	out, err := cmd.Output()
-	if err != nil {
-		return out, errors.New(stderr.String())
-	}
-
-	return out, nil
-}
-
-func setComposeFile(stack *portainer.Stack) []string {
-	options := make([]string, 0)
-
-	if stack == nil || stack.EntryPoint == "" {
-		return options
-	}
-
-	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
-	options = append(options, "-f", composeFilePath)
-	return options
-}
-
-func addProjectNameOption(options []string, stack *portainer.Stack) []string {
-	if stack == nil || stack.Name == "" {
-		return options
-	}
-
-	options = append(options, "-p", stack.Name)
-	return options
-}
-
-func addEnvFileOption(options []string, stack *portainer.Stack) ([]string, error) {
-	if stack == nil || stack.Env == nil || len(stack.Env) == 0 {
-		return options, nil
-	}
-
-	envFilePath := path.Join(stack.ProjectPath, "stack.env")
-
-	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return options, err
-	}
-
-	for _, v := range stack.Env {
-		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
-	}
-	envfile.Close()
-
-	options = append(options, "--env-file", envFilePath)
-	return options, nil
-}

api/exec/compose_wrapper_test.go

@@ -1,143 +0,0 @@
-package exec
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_setComposeFile(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected []string
-	}{
-		{
-			name:     "should return empty result if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should return empty result if stack don't have entrypoint",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should allow file name and dir",
-			stack: &portainer.Stack{
-				ProjectPath: "dir",
-				EntryPoint:  "file",
-			},
-			expected: []string{"-f", path.Join("dir", "file")},
-		},
-		{
-			name: "should allow file name only",
-			stack: &portainer.Stack{
-				EntryPoint: "file",
-			},
-			expected: []string{"-f", "file"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := setComposeFile(tt.stack)
-			assert.ElementsMatch(t, tt.expected, result)
-		})
-	}
-}
-
-func Test_addProjectNameOption(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected []string
-	}{
-		{
-			name:     "should not add project option if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should not add project option if stack doesn't have name",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should add project name option if stack has a name",
-			stack: &portainer.Stack{
-				Name: "project-name",
-			},
-			expected: []string{"-p", "project-name"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			options := []string{"-a", "b"}
-			result := addProjectNameOption(options, tt.stack)
-			assert.ElementsMatch(t, append(options, tt.expected...), result)
-		})
-	}
-}
-
-func Test_addEnvFileOption(t *testing.T) {
-	dir := t.TempDir()
-
-	tests := []struct {
-		name            string
-		stack           *portainer.Stack
-		expected        []string
-		expectedContent string
-	}{
-		{
-			name:     "should not add env file option if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should not add env file option if stack doesn't have env variables",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should not add env file option if stack's env variables are empty",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env:         []portainer.Pair{},
-			},
-			expected: []string{},
-		},
-		{
-			name: "should add env file option if stack has env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env: []portainer.Pair{
-					{Name: "var1", Value: "value1"},
-					{Name: "var2", Value: "value2"},
-				},
-			},
-			expected:        []string{"--env-file", path.Join(dir, "stack.env")},
-			expectedContent: "var1=value1\nvar2=value2\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			options := []string{"-a", "b"}
-			result, _ := addEnvFileOption(options, tt.stack)
-			assert.ElementsMatch(t, append(options, tt.expected...), result)
-
-			if tt.expectedContent != "" {
-				f, _ := os.Open(path.Join(dir, "stack.env"))
-				content, _ := ioutil.ReadAll(f)
-
-				assert.Equal(t, tt.expectedContent, string(content))
-			}
-		})
-	}
-}

api/exec/kubernetes_deploy.go

@@ -2,71 +2,235 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
+	"time"
+
+	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/kubernetes/cli"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath string
+	binaryPath                  string
+	dataStore                   portainer.DataStore
+	reverseTunnelService        portainer.ReverseTunnelService
+	signatureService            portainer.DigitalSignatureService
+	kubernetesClientFactory     *cli.ClientFactory
+	kubernetesTokenCacheManager *kubernetes.TokenCacheManager
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath: binaryPath,
+		binaryPath:                  binaryPath,
+		dataStore:                   datastore,
+		reverseTunnelService:        reverseTunnelService,
+		signatureService:            signatureService,
+		kubernetesClientFactory:     kubernetesClientFactory,
+		kubernetesTokenCacheManager: kubernetesTokenCacheManager,
 	}
 }
 
+func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
+	tokenData, err := security.RetrieveTokenData(request)
+	if err != nil {
+		return "", err
+	}
+
+	kubecli, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return "", err
+	}
+
+	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(int(endpoint.ID))
+
+	tokenManager, err := kubernetes.NewTokenManager(kubecli, deployer.dataStore, tokenCache, setLocalAdminToken)
+	if err != nil {
+		return "", err
+	}
+
+	if tokenData.Role == portainer.AdministratorRole {
+		return tokenManager.GetAdminServiceAccountToken(), nil
+	}
+
+	token, err := tokenManager.GetUserServiceAccountToken(int(tokenData.ID), endpoint.ID)
+	if err != nil {
+		return "", err
+	}
+
+	if token == "" {
+		return "", fmt.Errorf("can not get a valid user service account token")
+	}
+	return token, nil
+}
+
 // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
-// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
 // Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
-	if composeFormat {
-		convertedData, err := deployer.convertComposeData(data)
+func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
+	if endpoint.Type == portainer.KubernetesLocalEnvironment {
+		token, err := deployer.getToken(request, endpoint, true)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		data = string(convertedData)
+
+		command := path.Join(deployer.binaryPath, "kubectl")
+		if runtime.GOOS == "windows" {
+			command = path.Join(deployer.binaryPath, "kubectl.exe")
+		}
+
+		args := make([]string, 0)
+		args = append(args, "--server", endpoint.URL)
+		args = append(args, "--insecure-skip-tls-verify")
+		args = append(args, "--token", token)
+		args = append(args, "--namespace", namespace)
+		args = append(args, "apply", "-f", "-")
+
+		var stderr bytes.Buffer
+		cmd := exec.Command(command, args...)
+		cmd.Stderr = &stderr
+		cmd.Stdin = strings.NewReader(stackConfig)
+
+		output, err := cmd.Output()
+		if err != nil {
+			return "", errors.New(stderr.String())
+		}
+
+		return string(output), nil
 	}
 
-	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+	// agent
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		if tunnel.Status == portainer.EdgeAgentIdle {
+
+			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+			if err != nil {
+				return "", err
+			}
+
+			settings, err := deployer.dataStore.Settings().Settings()
+			if err != nil {
+				return "", err
+			}
+
+			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
+			time.Sleep(waitForAgentToConnect * 2)
+		}
+
+		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+	}
+
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return "", err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	httpCli := &http.Client{
+		Transport: transport,
+	}
+
+	if !strings.HasPrefix(endpointURL, "http") {
+		endpointURL = fmt.Sprintf("https://%s", endpointURL)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	command := path.Join(deployer.binaryPath, "kubectl")
-	if runtime.GOOS == "windows" {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
-	}
-
-	args := make([]string, 0)
-	args = append(args, "--server", endpoint.URL)
-	args = append(args, "--insecure-skip-tls-verify")
-	args = append(args, "--token", string(token))
-	args = append(args, "--namespace", namespace)
-	args = append(args, "apply", "-f", "-")
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = &stderr
-	cmd.Stdin = strings.NewReader(data)
-
-	output, err := cmd.Output()
+	reqPayload, err := json.Marshal(
+		struct {
+			StackConfig string
+			Namespace   string
+		}{
+			StackConfig: stackConfig,
+			Namespace:   namespace,
+		})
 	if err != nil {
-		return nil, errors.New(stderr.String())
+		return "", err
 	}
 
-	return output, nil
+	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
+	if err != nil {
+		return "", err
+	}
+
+	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := deployer.getToken(request, endpoint, false)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
+	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+	req.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var errorResponseData struct {
+			Message string
+			Details string
+		}
+		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
+		if err != nil {
+			output, parseStringErr := ioutil.ReadAll(resp.Body)
+			if parseStringErr != nil {
+				return "", parseStringErr
+			}
+
+			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
+
+		}
+
+		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
+	}
+
+	var responseData struct{ Output string }
+	err = json.NewDecoder(resp.Body).Decode(&responseData)
+	if err != nil {
+		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
+		if parseStringErr != nil {
+			return "", parseStringErr
+		}
+
+		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
+	}
+
+	return responseData.Output, nil
+
 }
 
-func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
+// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
+func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")
@@ -78,7 +242,7 @@ func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, err
 	var stderr bytes.Buffer
 	cmd := exec.Command(command, args...)
 	cmd.Stderr = &stderr
-	cmd.Stdin = strings.NewReader(data)
+	cmd.Stdin = bytes.NewReader(data)
 
 	output, err := cmd.Output()
 	if err != nil {

api/exec/swarm_stack.go

@@ -8,15 +8,18 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"runtime"
+	"strings"
 
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
 // SwarmStackManager represents a service for managing stacks.
 type SwarmStackManager struct {
 	binaryPath           string
-	dataPath             string
+	configPath           string
 	signatureService     portainer.DigitalSignatureService
 	fileService          portainer.FileService
 	reverseTunnelService portainer.ReverseTunnelService
@@ -24,16 +27,16 @@ type SwarmStackManager struct {
 
 // NewSwarmStackManager initializes a new SwarmStackManager service.
 // It also updates the configuration of the Docker CLI binary.
-func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
+func NewSwarmStackManager(binaryPath, configPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
 	manager := &SwarmStackManager{
 		binaryPath:           binaryPath,
-		dataPath:             dataPath,
+		configPath:           configPath,
 		signatureService:     signatureService,
 		fileService:          fileService,
 		reverseTunnelService: reverseTunnelService,
 	}
 
-	err := manager.updateDockerCLIConfiguration(dataPath)
+	err := manager.updateDockerCLIConfiguration(manager.configPath)
 	if err != nil {
 		return nil, err
 	}
@@ -42,51 +45,47 @@ func NewSwarmStackManager(binaryPath, dataPath string, signatureService portaine
 }
 
 // Login executes the docker login command against a list of registries (including DockerHub).
-func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
-	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoint *portainer.Endpoint) {
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	for _, registry := range registries {
 		if registry.Authentication {
 			registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL)
 			runCommandAndCaptureStdErr(command, registryArgs, nil, "")
 		}
 	}
-
-	if dockerhub.Authentication {
-		dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password)
-		runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "")
-	}
 }
 
 // Logout executes the docker logout command.
 func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
-	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	args = append(args, "logout")
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
-	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
-	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	filePaths := stackutils.GetStackFilePaths(stack)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 
 	if prune {
-		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth")
 	} else {
-		args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--with-registry-auth")
 	}
 
+	args = configureFilePaths(args, filePaths)
+	args = append(args, stack.Name)
+
 	env := make([]string, 0)
 	for _, envvar := range stack.Env {
 		env = append(env, envvar.Name+"="+envvar.Value)
 	}
-
-	stackFolder := path.Dir(stackFilePath)
-	return runCommandAndCaptureStdErr(command, args, env, stackFolder)
+	return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath)
 }
 
 // Remove executes the docker stack rm command.
 func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	args = append(args, "stack", "rm", stack.Name)
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
@@ -110,7 +109,7 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor
 	return nil
 }
 
-func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
+func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, configPath string, endpoint *portainer.Endpoint) (string, []string) {
 	// Assume Linux as a default
 	command := path.Join(binaryPath, "docker")
 
@@ -119,7 +118,7 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPa
 	}
 
 	args := make([]string, 0)
-	args = append(args, "--config", dataPath)
+	args = append(args, "--config", configPath)
 
 	endpointURL := endpoint.URL
 	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
@@ -146,8 +145,8 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPa
 	return command, args
 }
 
-func (manager *SwarmStackManager) updateDockerCLIConfiguration(dataPath string) error {
-	configFilePath := path.Join(dataPath, "config.json")
+func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string) error {
+	configFilePath := path.Join(configPath, "config.json")
 	config, err := manager.retrieveConfigurationFromDisk(configFilePath)
 	if err != nil {
 		return err
@@ -189,3 +188,15 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 
 	return config, nil
 }
+
+func (manager *SwarmStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
+}
+
+func configureFilePaths(args []string, filePaths []string) []string {
+	for _, path := range filePaths {
+		args = append(args, "--compose-file", path)
+	}
+	return args
+}

api/exec/swarm_stack_test.go

@@ -0,0 +1,15 @@
+package exec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigFilePaths(t *testing.T) {
+	args := []string{"stack", "deploy", "--with-registry-auth"}
+	filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"}
+	expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"}
+	output := configureFilePaths(args, filePaths)
+	assert.ElementsMatch(t, expected, output, "wrong output file paths")
+}

api/exec/utils.go

@@ -1,24 +0,0 @@
-package exec
-
-import (
-	"os/exec"
-	"path/filepath"
-	"runtime"
-)
-
-func osProgram(program string) string {
-	if runtime.GOOS == "windows" {
-		program += ".exe"
-	}
-	return program
-}
-
-func programPath(rootPath, program string) string {
-	return filepath.Join(rootPath, osProgram(program))
-}
-
-// IsBinaryPresent returns true if corresponding program exists on PATH
-func IsBinaryPresent(program string) bool {
-	_, err := exec.LookPath(program)
-	return err == nil
-}

api/exec/utils_test.go

@@ -1,16 +0,0 @@
-package exec
-
-import (
-	"testing"
-)
-
-func Test_isBinaryPresent(t *testing.T) {
-
-	if !IsBinaryPresent("docker") {
-		t.Error("expect docker binary to exist on the path")
-	}
-
-	if IsBinaryPresent("executable-with-this-name-should-not-exist") {
-		t.Error("expect binary with a random name to be missing on the path")
-	}
-}

api/filesystem/copy.go

@@ -1,4 +1,4 @@
-package backup
+package filesystem
 
 import (
 	"errors"
@@ -8,7 +8,8 @@ import (
 	"strings"
 )
 
-func copyPath(path string, toDir string) error {
+// CopyPath copies file or directory defined by the path to the toDir path
+func CopyPath(path string, toDir string) error {
 	info, err := os.Stat(path)
 	if err != nil && errors.Is(err, os.ErrNotExist) {
 		// skip copy if file does not exist
@@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error {
 		return copyFile(path, destination)
 	}
 
-	return copyDir(path, toDir)
+	return CopyDir(path, toDir, true)
 }
 
-func copyDir(fromDir, toDir string) error {
+// CopyDir copies contents of fromDir to toDir.
+// When keepParent is true, contents will be copied with their immediate parent dir,
+// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/<children>
+func CopyDir(fromDir, toDir string, keepParent bool) error {
 	cleanedSourcePath := filepath.Clean(fromDir)
 	parentDirectory := filepath.Dir(cleanedSourcePath)
 	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		var destination string
+		if keepParent {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		} else {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath))
+		}
+
+		if destination == "" {
+			return nil
+		}
+
 		if info.IsDir() {
 			return nil // skip directory creations
 		}

api/filesystem/copy_test.go

@@ -0,0 +1,92 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyFile("does-not-exist", tmpdir)
+	assert.Error(t, err)
+}
+
+func Test_copyFile_shouldMakeAbackup(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+
+	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
+	assert.NoError(t, err)
+
+	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, true)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}
+
+func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, false)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "outer"))
+	assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "dir", "inner"))
+}
+
+func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := CopyPath("does-not-exists", tmpdir)
+	assert.NoError(t, err)
+
+	assert.NoFileExists(t, tmpdir)
+}
+
+func Test_CopyPath_shouldCopyFile(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+
+	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
+	err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
+	assert.NoError(t, err)
+
+	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	assert.NoError(t, err)
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyPath_shouldCopyDir(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyPath("./testdata/copy_test", destination)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}

api/filesystem/filesystem.go

@@ -31,6 +31,8 @@ const (
 	ComposeStorePath = "compose"
 	// ComposeFileDefaultName represents the default name of a compose file.
 	ComposeFileDefaultName = "docker-compose.yml"
+	// ManifestFileDefaultName represents the default name of a k8s manifest file.
+	ManifestFileDefaultName = "k8s-deployment.yml"
 	// EdgeStackStorePath represents the subfolder where edge stack files are stored in the file store folder.
 	EdgeStackStorePath = "edge_stacks"
 	// PrivateKeyFile represents the name on disk of the file containing the private key.
@@ -41,6 +43,8 @@ const (
 	BinaryStorePath = "bin"
 	// EdgeJobStorePath represents the subfolder where schedule files are stored.
 	EdgeJobStorePath = "edge_jobs"
+	// DockerConfigPath represents the subfolder where docker configuration is stored.
+	DockerConfigPath = "docker_config"
 	// ExtensionRegistryManagementStorePath represents the subfolder where files related to the
 	// registry management extension are stored.
 	ExtensionRegistryManagementStorePath = "extensions"
@@ -48,6 +52,12 @@ const (
 	CustomTemplateStorePath = "custom_templates"
 	// TempPath represent the subfolder where temporary files are saved
 	TempPath = "tmp"
+	// SSLCertPath represents the default ssl certificates path
+	SSLCertPath = "certs"
+	// DefaultSSLCertFilename represents the default ssl certificate file name
+	DefaultSSLCertFilename = "cert.pem"
+	// DefaultSSLKeyFilename represents the default ssl key file name
+	DefaultSSLKeyFilename = "key.pem"
 )
 
 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -72,6 +82,11 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
+	err = service.createDirectoryInStore(SSLCertPath)
+	if err != nil {
+		return nil, err
+	}
+
 	err = service.createDirectoryInStore(TLSStorePath)
 	if err != nil {
 		return nil, err
@@ -87,6 +102,11 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
+	err = service.createDirectoryInStore(DockerConfigPath)
+	if err != nil {
+		return nil, err
+	}
+
 	return service, nil
 }
 
@@ -95,6 +115,11 @@ func (service *Service) GetBinaryFolder() string {
 	return path.Join(service.fileStorePath, BinaryStorePath)
 }
 
+// GetDockerConfigPath returns the full path to the docker config store on the filesystem
+func (service *Service) GetDockerConfigPath() string {
+	return path.Join(service.fileStorePath, DockerConfigPath)
+}
+
 // RemoveDirectory removes a directory on the filesystem.
 func (service *Service) RemoveDirectory(directoryPath string) error {
 	return os.RemoveAll(directoryPath)
@@ -106,6 +131,66 @@ func (service *Service) GetStackProjectPath(stackIdentifier string) string {
 	return path.Join(service.fileStorePath, ComposeStorePath, stackIdentifier)
 }
 
+// Copy copies the file on fromFilePath to toFilePath
+// if toFilePath exists func will fail unless deleteIfExists is true
+func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error {
+	exists, err := service.FileExists(fromFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return errors.New("File doesn't exist")
+	}
+
+	finput, err := os.Open(fromFilePath)
+	if err != nil {
+		return err
+	}
+
+	defer finput.Close()
+
+	exists, err = service.FileExists(toFilePath)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		if !deleteIfExists {
+			return errors.New("Destination file exists")
+		}
+
+		err := os.Remove(toFilePath)
+		if err != nil {
+			return err
+		}
+	}
+
+	foutput, err := os.Create(toFilePath)
+	if err != nil {
+		return err
+	}
+
+	defer foutput.Close()
+
+	buf := make([]byte, 1024)
+	for {
+		n, err := finput.Read(buf)
+		if err != nil && err != io.EOF {
+			return err
+		}
+		if n == 0 {
+			break
+		}
+
+		if _, err := foutput.Write(buf[:n]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // StoreStackFileFromBytes creates a subfolder in the ComposeStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
@@ -279,13 +364,7 @@ func (service *Service) WriteJSONToFile(path string, content interface{}) error
 
 // FileExists checks for the existence of the specified file.
 func (service *Service) FileExists(filePath string) (bool, error) {
-	if _, err := os.Stat(filePath); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
+	return FileExists(filePath)
 }
 
 // KeyPairFilesExist checks for the existence of the key files.
@@ -510,3 +589,83 @@ func (service *Service) GetTemporaryPath() (string, error) {
 func (service *Service) GetDatastorePath() string {
 	return service.dataStorePath
 }
+
+func (service *Service) wrapFileStore(filepath string) string {
+	return path.Join(service.fileStorePath, filepath)
+}
+
+func defaultCertPathUnderFileStore() (string, string) {
+	certPath := path.Join(SSLCertPath, DefaultSSLCertFilename)
+	keyPath := path.Join(SSLCertPath, DefaultSSLKeyFilename)
+	return certPath, keyPath
+}
+
+// GetDefaultSSLCertsPath returns the ssl certs path
+func (service *Service) GetDefaultSSLCertsPath() (string, string) {
+	certPath, keyPath := defaultCertPathUnderFileStore()
+	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath)
+}
+
+// StoreSSLCertPair stores a ssl certificate pair
+func (service *Service) StoreSSLCertPair(cert, key []byte) (string, string, error) {
+	certPath, keyPath := defaultCertPathUnderFileStore()
+
+	r := bytes.NewReader(cert)
+	err := service.createFileInStore(certPath, r)
+	if err != nil {
+		return "", "", err
+	}
+
+	r = bytes.NewReader(key)
+	err = service.createFileInStore(keyPath, r)
+	if err != nil {
+		return "", "", err
+	}
+
+	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath), nil
+}
+
+// CopySSLCertPair copies a ssl certificate pair
+func (service *Service) CopySSLCertPair(certPath, keyPath string) (string, string, error) {
+	defCertPath, defKeyPath := service.GetDefaultSSLCertsPath()
+
+	err := service.Copy(certPath, defCertPath, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	err = service.Copy(keyPath, defKeyPath, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	return defCertPath, defKeyPath, nil
+}
+
+// FileExists checks for the existence of the specified file.
+func FileExists(filePath string) (bool, error) {
+	if _, err := os.Stat(filePath); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func MoveDirectory(originalPath, newPath string) error {
+	if _, err := os.Stat(originalPath); err != nil {
+		return err
+	}
+
+	alreadyExists, err := FileExists(newPath)
+	if err != nil {
+		return err
+	}
+
+	if alreadyExists {
+		return errors.New("Target path already exists")
+	}
+
+	return os.Rename(originalPath, newPath)
+}

api/filesystem/filesystem_fileexists_test.go

@@ -0,0 +1,55 @@
+package filesystem
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_fileSystemService_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
+	service := createService(t)
+	testHelperFileExists_fileExists(t, service.FileExists)
+}
+
+func Test_fileSystemService_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
+	service := createService(t)
+	testHelperFileExists_fileNotExists(t, service.FileExists)
+}
+
+func Test_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
+	testHelperFileExists_fileExists(t, FileExists)
+}
+
+func Test_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
+	testHelperFileExists_fileNotExists(t, FileExists)
+}
+
+func testHelperFileExists_fileExists(t *testing.T, checker func(path string) (bool, error)) {
+	file, err := os.CreateTemp("", t.Name())
+	assert.NoError(t, err, "CreateTemp should not fail")
+
+	t.Cleanup(func() {
+		os.RemoveAll(file.Name())
+	})
+
+	exists, err := checker(file.Name())
+	assert.NoError(t, err, "FileExists should not fail")
+
+	assert.True(t, exists)
+}
+
+func testHelperFileExists_fileNotExists(t *testing.T, checker func(path string) (bool, error)) {
+	filePath := path.Join(os.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int()))
+
+	err := os.RemoveAll(filePath)
+	assert.NoError(t, err, "RemoveAll should not fail")
+
+	exists, err := checker(filePath)
+	assert.NoError(t, err, "FileExists should not fail")
+
+	assert.False(t, exists)
+}

api/filesystem/filesystem_move_test.go

@@ -0,0 +1,49 @@
+package filesystem
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// temporary function until upgrade to 1.16
+func tempDir(t *testing.T) string {
+	tmpDir, err := os.MkdirTemp("", "dir")
+	assert.NoError(t, err, "MkdirTemp should not fail")
+
+	return tmpDir
+}
+
+func Test_movePath_shouldFailIfOriginalPathDoesntExist(t *testing.T) {
+	tmpDir := tempDir(t)
+	missingPath := path.Join(tmpDir, "missing")
+	targetPath := path.Join(tmpDir, "target")
+
+	defer os.RemoveAll(tmpDir)
+
+	err := MoveDirectory(missingPath, targetPath)
+	assert.Error(t, err, "move directory should fail when target path exists")
+}
+
+func Test_movePath_shouldFailIfTargetPathDoesExist(t *testing.T) {
+	originalPath := tempDir(t)
+	missingPath := tempDir(t)
+
+	defer os.RemoveAll(originalPath)
+	defer os.RemoveAll(missingPath)
+
+	err := MoveDirectory(originalPath, missingPath)
+	assert.Error(t, err, "move directory should fail when target path exists")
+}
+
+func Test_movePath_success(t *testing.T) {
+	originalPath := tempDir(t)
+
+	defer os.RemoveAll(originalPath)
+
+	err := MoveDirectory(originalPath, fmt.Sprintf("%s-old", originalPath))
+	assert.NoError(t, err)
+}

api/filesystem/filesystem_test.go

@@ -0,0 +1,22 @@
+package filesystem
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createService(t *testing.T) *Service {
+	dataStorePath := path.Join(os.TempDir(), t.Name())
+
+	service, err := NewService(dataStorePath, "")
+	assert.NoError(t, err, "NewService should not fail")
+
+	t.Cleanup(func() {
+		os.RemoveAll(dataStorePath)
+	})
+
+	return service
+}

api/git/azure.go

@@ -2,15 +2,17 @@ package git
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
 )
 
 const (
@@ -37,7 +39,7 @@ type azureDownloader struct {
 
 func NewAzureDownloader(client *http.Client) *azureDownloader {
 	return &azureDownloader{
-		client: client,
+		client:  client,
 		baseUrl: "https://dev.azure.com",
 	}
 }
@@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	return zipFile.Name(), nil
 }
 
+func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
+	config, err := parseUrl(options.repositoryUrl)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to parse url")
+	}
+
+	refsUrl, err := a.buildRefsUrl(config, options.referenceName)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to build azure refs url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil)
+	if options.username != "" || options.password != "" {
+		req.SetBasicAuth(options.username, options.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status)
+	}
+
+	var refs struct {
+		Value []struct {
+			Name     string `json:"name"`
+			ObjectId string `json:"objectId"`
+		}
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
+		return "", errors.Wrap(err, "could not parse Azure Refs response")
+	}
+
+	for _, ref := range refs.Value {
+		if strings.EqualFold(ref.Name, options.referenceName) {
+			return ref.ObjectId, nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", options.referenceName)
+}
+
 func parseUrl(rawUrl string) (*azureOptions, error) {
 	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
 		return parseHttpUrl(rawUrl)
@@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
+func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) {
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl)
+	}
+
+	// filterContains=main&api-version=6.0
+	q := u.Query()
+	q.Set("filterContains", formatReferenceName(referenceName))
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
 const (
 	branchPrefix = "refs/heads/"
 	tagPrefix    = "refs/tags/"

api/git/azure_integration_test.go

@@ -2,12 +2,13 @@ package git
 
 import (
 	"fmt"
-	"github.com/docker/docker/pkg/ioutils"
-	_ "github.com/joho/godotenv/autoload"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestService_ClonePublicRepository_Azure(t *testing.T) {
@@ -54,7 +55,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 			assert.NoError(t, err)
 			defer os.RemoveAll(dst)
 			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
-			err = service.ClonePublicRepository(repositoryUrl, tt.args.referenceName, dst)
+			err = service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "")
 			assert.NoError(t, err)
 			assert.FileExists(t, filepath.Join(dst, "README.md"))
 		})
@@ -72,11 +73,23 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 	defer os.RemoveAll(dst)
 
 	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
-	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, "", pat)
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
 
+func TestService_LatestCommitID_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}
+
 func getRequiredValue(t *testing.T, name string) string {
 	value, ok := os.LookupEnv(name)
 	if !ok {

api/git/azure_test.go

@@ -2,11 +2,12 @@ package git
 
 import (
 	"context"
-	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_buildDownloadUrl(t *testing.T) {
@@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) {
 	}
 }
 
+func Test_buildRefsUrl(t *testing.T) {
+	a := NewAzureDownloader(nil)
+	u, err := a.buildRefsUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "refs/heads/main")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
 func Test_parseAzureUrl(t *testing.T) {
 	type args struct {
 		url string
@@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		})
 	}
 }
+
+func Test_azureDownloader_latestCommitID(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		response := `{
+			"value": [
+				{
+					"name": "refs/heads/feature/calcApp",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp"
+				},
+				{
+					"name": "refs/heads/feature/replacer",
+					"objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer"
+				},
+				{
+					"name": "refs/heads/master",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster"
+				}
+			],
+			"count": 3
+		}`
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(response))
+	}))
+	defer server.Close()
+
+	a := &azureDownloader{
+		client:  server.Client(),
+		baseUrl: server.URL,
+	}
+
+	tests := []struct {
+		name    string
+		args    fetchOptions
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/master",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "ffe9cba521f00d7f60e322845072238635edb451",
+			wantErr: false,
+		},
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/unknown",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id, err := a.latestCommitID(context.Background(), tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, id)
+		})
+	}
+}

api/git/git.go

@@ -3,18 +3,29 @@ package git
 import (
 	"context"
 	"crypto/tls"
-	"github.com/pkg/errors"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/transport/client"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/storage/memory"
 )
 
+type fetchOptions struct {
+	repositoryUrl string
+	username      string
+	password      string
+	referenceName string
+}
+
 type cloneOptions struct {
 	repositoryUrl string
 	username      string
@@ -25,9 +36,10 @@ type cloneOptions struct {
 
 type downloader interface {
 	download(ctx context.Context, dst string, opt cloneOptions) error
+	latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
 }
 
-type gitClient struct{
+type gitClient struct {
 	preserveGitDirectory bool
 }
 
@@ -35,13 +47,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	gitOptions := git.CloneOptions{
 		URL:   opt.repositoryUrl,
 		Depth: opt.depth,
-	}
-
-	if opt.password != "" || opt.username != "" {
-		gitOptions.Auth = &githttp.BasicAuth{
-			Username: opt.username,
-			Password: opt.password,
-		}
+		Auth:  getAuth(opt.username, opt.password),
 	}
 
 	if opt.referenceName != "" {
@@ -61,6 +67,44 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	return nil
 }
 
+func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
+	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
+		Name: "origin",
+		URLs: []string{opt.repositoryUrl},
+	})
+
+	listOptions := &git.ListOptions{
+		Auth: getAuth(opt.username, opt.password),
+	}
+
+	refs, err := remote.List(listOptions)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to list repository refs")
+	}
+
+	for _, ref := range refs {
+		if strings.EqualFold(ref.Name().String(), opt.referenceName) {
+			return ref.Hash().String(), nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName)
+}
+
+func getAuth(username, password string) *githttp.BasicAuth {
+	if password != "" {
+		if username == "" {
+			username = "token"
+		}
+
+		return &githttp.BasicAuth{
+			Username: username,
+			Password: password,
+		}
+	}
+	return nil
+}
+
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
@@ -73,6 +117,7 @@ func NewService() *Service {
 	httpsCli := &http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			Proxy:           http.ProxyFromEnvironment,
 		},
 		Timeout: 300 * time.Second,
 	}
@@ -86,26 +131,18 @@ func NewService() *Service {
 	}
 }
 
-// ClonePublicRepository clones a public git repository using the specified URL in the specified
+// CloneRepository clones a git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, referenceName, destination string) error {
-	return service.cloneRepository(destination, cloneOptions{
-		repositoryUrl: repositoryURL,
-		referenceName: referenceName,
-		depth:         1,
-	})
-}
-
-// ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
-// destination folder. It will use the specified Username and Password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName, destination, username, password string) error {
-	return service.cloneRepository(destination, cloneOptions{
+func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	options := cloneOptions{
 		repositoryUrl: repositoryURL,
 		username:      username,
 		password:      password,
 		referenceName: referenceName,
 		depth:         1,
-	})
+	}
+
+	return service.cloneRepository(destination, options)
 }
 
 func (service *Service) cloneRepository(destination string, options cloneOptions) error {
@@ -115,3 +152,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions
 
 	return service.git.download(context.TODO(), destination, options)
 }
+
+// LatestCommitID returns SHA1 of the latest commit of the specified reference
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	options := fetchOptions{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+		referenceName: referenceName,
+	}
+
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.latestCommitID(context.TODO(), options)
+	}
+
+	return service.git.latestCommitID(context.TODO(), options)
+}

api/git/git_integration_test.go

@@ -1,17 +1,18 @@
 package git
 
 import (
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	ensureIntegrationTest(t)
 
-	pat := getRequiredValue(t, "GITHUB_PAT")
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := NewService()
 
@@ -20,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	defer os.RemoveAll(dst)
 
 	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
-	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, username, pat)
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
+
+func TestService_LatestCommitID_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService()
+
+	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}

api/git/git_test.go

@@ -2,16 +2,17 @@ package git
 
 import (
 	"context"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/stretchr/testify/assert"
 )
 
 var bareRepoDir string
@@ -59,7 +60,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 	t.Logf("Cloning into %s", dir)
-	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
+	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
 	assert.NoError(t, err)
 	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
 }
@@ -74,9 +75,11 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to create a temp dir")
 	}
+
 	defer os.RemoveAll(dir)
+
 	t.Logf("Cloning into %s", dir)
-	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
+	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
 	assert.NoError(t, err)
 	assert.NoDirExists(t, filepath.Join(dir, ".git"))
 }
@@ -102,7 +105,19 @@ func Test_cloneRepository(t *testing.T) {
 	})
 
 	assert.NoError(t, err)
-	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+	assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func Test_latestCommitID(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+
+	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "")
+
+	assert.NoError(t, err)
+	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
 }
 
 func getCommitHistoryLength(t *testing.T, err error, dir string) int {
@@ -134,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e
 	return nil
 }
 
+func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
+	return "", nil
+}
+
 func Test_cloneRepository_azure(t *testing.T) {
 	tests := []struct {
 		name   string

api/git/types/types.go

@@ -0,0 +1,20 @@
+package gittypes
+
+// RepoConfig represents a configuration for a repo
+type RepoConfig struct {
+	// The repo url
+	URL string `example:"https://github.com/portainer/portainer.git"`
+	// The reference name
+	ReferenceName string `example:"refs/heads/branch_name"`
+	// Path to where the config file is in this url/refName
+	ConfigFilePath string `example:"docker-compose.yml"`
+	// Git credentials
+	Authentication *GitAuthentication
+	// Repository hash
+	ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
+}
+
+type GitAuthentication struct {
+	Username string
+	Password string
+}

api/go.mod

@@ -1,17 +1,23 @@
 module github.com/portainer/portainer/api
 
-go 1.13
+go 1.16
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.4.16
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/boltdb/bolt v1.3.1
 	github.com/containerd/containerd v1.3.1 // indirect
 	github.com/coreos/go-semver v0.3.0
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/cli v0.0.0-20191126203649-54d085b857e9
-	github.com/docker/docker v0.0.0-00010101000000-000000000000
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/g07cha/defender v0.0.0-20180505193036-5665c627c814
 	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-ldap/ldap/v3 v3.1.8
@@ -21,19 +27,28 @@ require (
 	github.com/gorilla/websocket v1.4.1
 	github.com/joho/godotenv v1.3.0
 	github.com/jpillora/chisel v0.0.0-20190724232113-f3a8df20e389
-	github.com/json-iterator/go v1.1.8
+	github.com/json-iterator/go v1.1.10
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
 	github.com/mattn/go-shellwords v1.0.6 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/libcompose v0.5.3
-	github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
+	github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1
+	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
+	github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
-	github.com/stretchr/testify v1.6.1
+	github.com/robfig/cron/v3 v3.0.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gotest.tools v2.2.0+incompatible // indirect
 	k8s.io/api v0.17.2
 	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v0.17.2

api/go.sum

@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
@@ -11,40 +11,31 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Microsoft/go-winio v0.3.8/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4=
+github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 h1:axBiC50cNZOs7ygH5BgQp4N+aYrZ2DNpWZ1KG3VOSOM=
 github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2/go.mod h1:jnzFpU88PccN/tPPhCpnNU8mZphvKxYM9lLNkd8e+os=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/containerd v1.3.1 h1:LdbWxLhkAIxGO7h3mATHkyav06WuDs/yTWxIljJOTks=
 github.com/containerd/containerd v1.3.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -56,23 +47,16 @@ github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfD
 github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/docker/cli v0.0.0-20190711175710-5b38d82aa076/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v0.0.0-20191126203649-54d085b857e9 h1:Q6D6b2iRKhvtL3Wj9p0SyPOvUDJ1ht62mbiBoNJ3Aus=
 github.com/docker/cli v0.0.0-20191126203649-54d085b857e9/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ=
-github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203 h1:QeBh8wW8pIZKlXxlMOQ8hSCMdJA+2Z/bD/iDyCAS8XU=
 github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
-github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
-github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
-github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA=
-github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
-github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
@@ -80,8 +64,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -101,20 +85,15 @@ github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbK
 github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
 github.com/go-git/go-git/v5 v5.3.0 h1:8WKMtJR2j8RntEXR/uvTKagfEt4GYlwQ7mntE4+0GWc=
 github.com/go-git/go-git/v5 v5.3.0/go.mod h1:xdX4bWJ48aOrdhnl2XqHYstHbbp6+LFS4r4X+lNVprw=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
 github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -123,7 +102,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -141,8 +119,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v0.0.0-20160317213430-0eeaf8392f5b/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
@@ -153,6 +129,7 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
@@ -172,25 +149,20 @@ github.com/jpillora/requestlog v0.0.0-20181015073026-df8817be5f82/go.mod h1:w8bu
 github.com/jpillora/sizestr v0.0.0-20160130011556-e2ea2fa42fb9 h1:0c9jcgBtHRtDU//jTrcCgWG6UHjMZytiq/3WhraNgUM=
 github.com/jpillora/sizestr v0.0.0-20160130011556-e2ea2fa42fb9/go.mod h1:1ffp+CRe0eAwwRb0/BownUAjMBsmTLwgAvRbfj9dRwE=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c h1:N7A4JCA2G+j5fuFxCsJqjFU/sZe0mj8H0sSoSwbaikw=
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c/go.mod h1:Nn5wlyECw3iJrzi0AhIWg+AJUb4PlRQVW4/3XHH1LZA=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v0.0.0-20150511174710-5cf931ef8f76/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -198,8 +170,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
@@ -211,58 +181,48 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
-github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420 h1:Yu3681ykYHDfLoI6XVjL4JWmkE+3TX9yfIWwRCh1kFM=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/image-spec v0.0.0-20170515205857-f03dbe35d449 h1:Aq8iG72akPb/kszE7ksZ5ldV+JYPYii/KZOxlpJF07s=
-github.com/opencontainers/image-spec v0.0.0-20170515205857-f03dbe35d449/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c h1:iOMba/KmaXgSX5PFKu1u6s+DZXiq+EzPayawa76w6aA=
-github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8=
-github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8=
-github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yHr4rtnirg0W0Cjvv6/DzxBIZk5sV59208=
-github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2/go.mod h1:/wIeGwJOMYc1JplE/OvYMO5korce39HddIfI8VKGyAM=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210909011155-9ff375eac059 h1:98v0k3x3ZXa09NaHP/HmSA83rcN8cuE/zTKo6xvNmoM=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210909011155-9ff375eac059/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1 h1:0ZGSu3Atz7RHMDsoITHV676igRfsb51mlgELGo37ELU=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
+github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
+github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
+github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97 h1:ZcRVgWHTac8V7WU9TUBr73H3e5ajVFYTPjPl9TWULDA=
+github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 h1:H8HR2dHdBf8HANSkUyVw4o8+4tegGcd+zyKZ3e599II=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33/go.mod h1:Y2TfgviWI4rT2qaOTHr+hq6MdKIE5YjgQAu7qwptTV0=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
-github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
-github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -273,21 +233,19 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
-github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -299,18 +257,15 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181017193950-04a2e542c03f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs=
@@ -323,8 +278,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
@@ -343,9 +296,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -355,9 +306,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM=
-google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -366,9 +316,11 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
@@ -377,13 +329,13 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.17.2 h1:NF1UFXcKN7/OOv1uxdRz3qfra8AHsPav5M93hlV9+Dc=
 k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4=
 k8s.io/apimachinery v0.17.2 h1:hwDQQFbdRlpnnsR64Asdi55GyCaIP/3WQpMmbNBeWr4=
@@ -395,6 +347,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=

api/http/errors/errors.go

@@ -4,7 +4,7 @@ import "errors"
 
 var (
 	// ErrEndpointAccessDenied Access denied to endpoint error
-	ErrEndpointAccessDenied = errors.New("Access denied to endpoint")
+	ErrEndpointAccessDenied = errors.New("Access denied to environment")
 	// ErrUnauthorized Unauthorized error
 	ErrUnauthorized = errors.New("Unauthorized")
 	// ErrResourceAccessDenied Access denied to resource error

api/http/handler/auth/authenticate.go

@@ -130,19 +130,13 @@ func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, use
 }
 
 func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
-	tokenData := &portainer.TokenData{
-		ID:       user.ID,
-		Username: user.Username,
-		Role:     user.Role,
-	}
-
-	return handler.persistAndWriteToken(w, tokenData)
+	return handler.persistAndWriteToken(w, composeTokenData(user))
 }
 
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
 	}
 
 	return response.JSON(w, &authenticateResponse{JWT: token})
@@ -204,3 +198,11 @@ func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamM
 	}
 	return false
 }
+
+func composeTokenData(user *portainer.User) *portainer.TokenData {
+	return &portainer.TokenData{
+		ID:       user.ID,
+		Username: user.Username,
+		Role:     user.Role,
+	}
+}

api/http/handler/auth/authenticate_oauth.go

@@ -25,17 +25,6 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id AuthenticateOauth
-// @summary Authenticate with OAuth
-// @tags auth
-// @accept json
-// @produce json
-// @param body body oauthPayload true "OAuth Credentials used for authentication"
-// @success 200 {object} authenticateResponse "Success"
-// @failure 400 "Invalid request"
-// @failure 422 "Invalid Credentials"
-// @failure 500 "Server error"
-// @router /auth/oauth/validate [post]
 func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, error) {
 	if code == "" {
 		return "", errors.New("Invalid OAuth authorization code")
@@ -53,35 +42,46 @@ func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuth
 	return username, nil
 }
 
+// @id ValidateOAuth
+// @summary Authenticate with OAuth
+// @tags auth
+// @accept json
+// @produce json
+// @param body body oauthPayload true "OAuth Credentials used for authentication"
+// @success 200 {object} authenticateResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 422 "Invalid Credentials"
+// @failure 500 "Server error"
+// @router /auth/oauth/validate [post]
 func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	var payload oauthPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve settings from the database", Err: err}
 	}
 
-	if settings.AuthenticationMethod != 3 {
-		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", errors.New("OAuth authentication is not enabled")}
+	if settings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "OAuth authentication is not enabled", Err: errors.New("OAuth authentication is not enabled")}
 	}
 
 	username, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
 	if err != nil {
 		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to authenticate through OAuth", Err: httperrors.ErrUnauthorized}
 	}
 
 	user, err := handler.DataStore.User().UserByUsername(username)
 	if err != nil && err != bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a user with the specified username from the database", Err: err}
 	}
 
 	if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
-		return &httperror.HandlerError{http.StatusForbidden, "Account not created beforehand in Portainer and automatic user provisioning not enabled", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Account not created beforehand in Portainer and automatic user provisioning not enabled", Err: httperrors.ErrUnauthorized}
 	}
 
 	if user == nil {
@@ -92,7 +92,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 		err = handler.DataStore.User().CreateUser(user)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist user inside the database", Err: err}
 		}
 
 		if settings.OAuthSettings.DefaultTeamID != 0 {
@@ -104,7 +104,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 			err = handler.DataStore.TeamMembership().CreateTeamMembership(membership)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
+				return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist team membership inside the database", Err: err}
 			}
 		}
 

api/http/handler/customtemplates/customtemplate_create.go

@@ -21,7 +21,7 @@ import (
 // @description **Access policy**: authenticated
 // @tags custom_templates
 // @security jwt
-// @accept json, multipart/form-data
+// @accept json,multipart/form-data
 // @produce json
 // @param method query string true "method for creating template" Enums(string, file, repository)
 // @param body_string body customTemplateFromFileContentPayload false "Required when using method=string"
@@ -105,9 +105,10 @@ type customTemplateFromFileContentPayload struct {
 	Note string `example:"This is my <b>custom</b> template"`
 	// Platform associated to the template.
 	// Valid values are: 1 - 'linux', 2 - 'windows'
-	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2" validate:"required"`
-	// Type of created stack (1 - swarm, 2 - compose)
-	Type portainer.StackType `example:"1" enums:"1,2" validate:"required"`
+	// Required for Docker stacks
+	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
+	// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
+	Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
 	// Content of stack file
 	FileContent string `validate:"required"`
 }
@@ -122,10 +123,10 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
 	if govalidator.IsNull(payload.FileContent) {
 		return errors.New("Invalid file content")
 	}
-	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
+	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
-	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
+	if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
 	return nil
@@ -171,7 +172,8 @@ type customTemplateFromGitRepositoryPayload struct {
 	Note string `example:"This is my <b>custom</b> template"`
 	// Platform associated to the template.
 	// Valid values are: 1 - 'linux', 2 - 'windows'
-	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2" validate:"required"`
+	// Required for Docker stacks
+	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
 	// Type of created stack (1 - swarm, 2 - compose)
 	Type portainer.StackType `example:"1" enums:"1,2" validate:"required"`
 
@@ -205,6 +207,11 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
 		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
 	}
+
+	if payload.Type == portainer.KubernetesStack {
+		return errors.New("Creating a Kubernetes custom template from git is not supported")
+	}
+
 	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
@@ -236,16 +243,14 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 	projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
 	customTemplate.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
 		return nil, err
 	}
@@ -280,20 +285,21 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 	note, _ := request.RetrieveMultiPartFormValue(r, "Note", true)
 	payload.Note = note
 
-	platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true)
-	templatePlatform := portainer.CustomTemplatePlatform(platform)
-	if templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows {
-		return errors.New("Invalid custom template platform")
-	}
-	payload.Platform = templatePlatform
-
 	typeNumeral, _ := request.RetrieveNumericMultiPartFormValue(r, "Type", true)
 	templateType := portainer.StackType(typeNumeral)
-	if templateType != portainer.DockerComposeStack && templateType != portainer.DockerSwarmStack {
+	if templateType != portainer.KubernetesStack && templateType != portainer.DockerSwarmStack && templateType != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
 	payload.Type = templateType
 
+	platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true)
+	templatePlatform := portainer.CustomTemplatePlatform(platform)
+	if templateType != portainer.KubernetesStack && templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows {
+		return errors.New("Invalid custom template platform")
+	}
+
+	payload.Platform = templatePlatform
+
 	composeFileContent, _, err := request.RetrieveMultiPartFormFile(r, "File")
 	if err != nil {
 		return errors.New("Invalid Compose file. Ensure that the Compose file is uploaded correctly")

api/http/handler/customtemplates/customtemplate_list.go

@@ -2,7 +2,9 @@ package customtemplates
 
 import (
 	"net/http"
+	"strconv"
 
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
@@ -17,10 +19,16 @@ import (
 // @tags custom_templates
 // @security jwt
 // @produce json
+// @param type query []int true "Template types" Enums(1,2,3)
 // @success 200 {array} portainer.CustomTemplate "Success"
 // @failure 500 "Server error"
 // @router /custom_templates [get]
 func (handler *Handler) customTemplateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	templateTypes, err := parseTemplateTypes(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid Custom template type", err}
+	}
+
 	customTemplates, err := handler.DataStore.CustomTemplate().CustomTemplates()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve custom templates from the database", err}
@@ -52,5 +60,52 @@ func (handler *Handler) customTemplateList(w http.ResponseWriter, r *http.Reques
 		customTemplates = authorization.FilterAuthorizedCustomTemplates(customTemplates, user, userTeamIDs)
 	}
 
+	customTemplates = filterByType(customTemplates, templateTypes)
+
 	return response.JSON(w, customTemplates)
 }
+
+func parseTemplateTypes(r *http.Request) ([]portainer.StackType, error) {
+	err := r.ParseForm()
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse request params")
+	}
+
+	types, exist := r.Form["type"]
+	if !exist {
+		return []portainer.StackType{}, nil
+	}
+
+	res := []portainer.StackType{}
+	for _, templateTypeStr := range types {
+		templateType, err := strconv.Atoi(templateTypeStr)
+		if err != nil {
+			return nil, errors.WithMessage(err, "failed parsing template type")
+		}
+
+		res = append(res, portainer.StackType(templateType))
+	}
+
+	return res, nil
+}
+
+func filterByType(customTemplates []portainer.CustomTemplate, templateTypes []portainer.StackType) []portainer.CustomTemplate {
+	if len(templateTypes) == 0 {
+		return customTemplates
+	}
+
+	typeSet := map[portainer.StackType]bool{}
+	for _, templateType := range templateTypes {
+		typeSet[templateType] = true
+	}
+
+	filtered := []portainer.CustomTemplate{}
+
+	for _, template := range customTemplates {
+		if typeSet[template.Type] {
+			filtered = append(filtered, template)
+		}
+	}
+
+	return filtered
+}

api/http/handler/customtemplates/customtemplate_update.go

@@ -27,9 +27,10 @@ type customTemplateUpdatePayload struct {
 	Note string `example:"This is my <b>custom</b> template"`
 	// Platform associated to the template.
 	// Valid values are: 1 - 'linux', 2 - 'windows'
-	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2" validate:"required"`
-	// Type of created stack (1 - swarm, 2 - compose)
-	Type portainer.StackType `example:"1" enums:"1,2" validate:"required"`
+	// Required for Docker stacks
+	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
+	// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
+	Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
 	// Content of stack file
 	FileContent string `validate:"required"`
 }
@@ -41,10 +42,10 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.FileContent) {
 		return errors.New("Invalid file content")
 	}
-	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
+	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
-	if payload.Type != portainer.DockerComposeStack && payload.Type != portainer.DockerSwarmStack {
+	if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
 	if govalidator.IsNull(payload.Description) {

api/http/handler/customtemplates/git.go

@@ -1,17 +0,0 @@
-package customtemplates
-
-type cloneRepositoryParameters struct {
-	url            string
-	referenceName  string
-	path           string
-	authentication bool
-	username       string
-	password       string
-}
-
-func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error {
-	if parameters.authentication {
-		return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password)
-	}
-	return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path)
-}

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -1,28 +0,0 @@
-package dockerhub
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-)
-
-// @id DockerHubInspect
-// @summary Retrieve DockerHub information
-// @description Use this endpoint to retrieve the information used to connect to the DockerHub
-// @description **Access policy**: authenticated
-// @tags dockerhub
-// @security jwt
-// @produce json
-// @success 200 {object} portainer.DockerHub
-// @failure 500 "Server error"
-// @router /dockerhub [get]
-func (handler *Handler) dockerhubInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
-	}
-
-	hideFields(dockerhub)
-	return response.JSON(w, dockerhub)
-}

api/http/handler/dockerhub/dockerhub_update.go

@@ -1,68 +0,0 @@
-package dockerhub
-
-import (
-	"errors"
-	"net/http"
-
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-)
-
-type dockerhubUpdatePayload struct {
-	// Enable authentication against DockerHub
-	Authentication bool `validate:"required" example:"false"`
-	// Username used to authenticate against the DockerHub
-	Username string `validate:"required" example:"hub_user"`
-	// Password used to authenticate against the DockerHub
-	Password string `validate:"required" example:"hub_password"`
-}
-
-func (payload *dockerhubUpdatePayload) Validate(r *http.Request) error {
-	if payload.Authentication && (govalidator.IsNull(payload.Username) || govalidator.IsNull(payload.Password)) {
-		return errors.New("Invalid credentials. Username and password must be specified when authentication is enabled")
-	}
-	return nil
-}
-
-// @id DockerHubUpdate
-// @summary Update DockerHub information
-// @description Use this endpoint to update the information used to connect to the DockerHub
-// @description **Access policy**: administrator
-// @tags dockerhub
-// @security jwt
-// @accept json
-// @produce json
-// @param body body dockerhubUpdatePayload true "DockerHub information"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @router /dockerhub [put]
-func (handler *Handler) dockerhubUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload dockerhubUpdatePayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
-	}
-
-	dockerhub := &portainer.DockerHub{
-		Authentication: false,
-		Username:       "",
-		Password:       "",
-	}
-
-	if payload.Authentication {
-		dockerhub.Authentication = true
-		dockerhub.Username = payload.Username
-		dockerhub.Password = payload.Password
-	}
-
-	err = handler.DataStore.DockerHub().UpdateDockerHub(dockerhub)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the Dockerhub changes inside the database", err}
-	}
-
-	return response.Empty(w)
-}

api/http/handler/dockerhub/handler.go

@@ -1,33 +0,0 @@
-package dockerhub
-
-import (
-	"net/http"
-
-	"github.com/gorilla/mux"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-func hideFields(dockerHub *portainer.DockerHub) {
-	dockerHub.Password = ""
-}
-
-// Handler is the HTTP handler used to handle DockerHub operations.
-type Handler struct {
-	*mux.Router
-	DataStore portainer.DataStore
-}
-
-// NewHandler creates a handler to manage Dockerhub operations.
-func NewHandler(bouncer *security.RequestBouncer) *Handler {
-	h := &Handler{
-		Router: mux.NewRouter(),
-	}
-	h.Handle("/dockerhub",
-		bouncer.RestrictedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
-	h.Handle("/dockerhub",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
-
-	return h
-}

api/http/handler/edgegroups/edgegroup_create.go

@@ -27,7 +27,7 @@ func (payload *edgeGroupCreatePayload) Validate(r *http.Request) error {
 		return errors.New("TagIDs is mandatory for a dynamic Edge group")
 	}
 	if !payload.Dynamic && (payload.Endpoints == nil || len(payload.Endpoints) == 0) {
-		return errors.New("Endpoints is mandatory for a static Edge group")
+		return errors.New("Environment is mandatory for a static Edge group")
 	}
 	return nil
 }
@@ -41,7 +41,7 @@ func (payload *edgeGroupCreatePayload) Validate(r *http.Request) error {
 // @produce json
 // @param body body edgeGroupCreatePayload true "EdgeGroup data"
 // @success 200 {object} portainer.EdgeGroup
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @failure 500
 // @router /edge_groups [post]
 func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -77,7 +77,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 		for _, endpointID := range payload.Endpoints {
 			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint from the database", err}
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environment from the database", err}
 			}
 
 			if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {

api/http/handler/edgegroups/edgegroup_delete.go

@@ -20,7 +20,7 @@ import (
 // @produce json
 // @param id path int true "EdgeGroup Id"
 // @success 204
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @failure 500
 // @router /edge_groups/{id} [delete]
 func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/edgegroups/edgegroup_inspect.go

@@ -19,7 +19,7 @@ import (
 // @produce json
 // @param id path int true "EdgeGroup Id"
 // @success 200 {object} portainer.EdgeGroup
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @failure 500
 // @router /edge_groups/{id} [get]
 func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -38,7 +38,7 @@ func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request)
 	if edgeGroup.Dynamic {
 		endpoints, err := handler.getEndpointsByTags(edgeGroup.TagIDs, edgeGroup.PartialMatch)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints and endpoint groups for Edge group", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments and environment groups for Edge group", err}
 		}
 
 		edgeGroup.Endpoints = endpoints

api/http/handler/edgegroups/edgegroup_list.go

@@ -1,6 +1,7 @@
 package edgegroups
 
 import (
+	"fmt"
 	"net/http"
 
 	httperror "github.com/portainer/libhttp/error"
@@ -10,7 +11,8 @@ import (
 
 type decoratedEdgeGroup struct {
 	portainer.EdgeGroup
-	HasEdgeStack bool `json:"HasEdgeStack"`
+	HasEdgeStack  bool `json:"HasEdgeStack"`
+	EndpointTypes []portainer.EndpointType
 }
 
 // @id EdgeGroupList
@@ -22,7 +24,7 @@ type decoratedEdgeGroup struct {
 // @produce json
 // @success 200 {array} portainer.EdgeGroup{HasEdgeStack=bool} "EdgeGroups"
 // @failure 500
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_groups [get]
 func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
@@ -46,17 +48,25 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h
 	decoratedEdgeGroups := []decoratedEdgeGroup{}
 	for _, orgEdgeGroup := range edgeGroups {
 		edgeGroup := decoratedEdgeGroup{
-			EdgeGroup: orgEdgeGroup,
+			EdgeGroup:     orgEdgeGroup,
+			EndpointTypes: []portainer.EndpointType{},
 		}
 		if edgeGroup.Dynamic {
-			endpoints, err := handler.getEndpointsByTags(edgeGroup.TagIDs, edgeGroup.PartialMatch)
+			endpointIDs, err := handler.getEndpointsByTags(edgeGroup.TagIDs, edgeGroup.PartialMatch)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints and endpoint groups for Edge group", err}
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments and environment groups for Edge group", err}
 			}
 
-			edgeGroup.Endpoints = endpoints
+			edgeGroup.Endpoints = endpointIDs
 		}
 
+		endpointTypes, err := getEndpointTypes(handler.DataStore.Endpoint(), edgeGroup.Endpoints)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint types for Edge group", err}
+		}
+
+		edgeGroup.EndpointTypes = endpointTypes
+
 		edgeGroup.HasEdgeStack = usedEdgeGroups[edgeGroup.ID]
 
 		decoratedEdgeGroups = append(decoratedEdgeGroups, edgeGroup)
@@ -64,3 +74,22 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h
 
 	return response.JSON(w, decoratedEdgeGroups)
 }
+
+func getEndpointTypes(endpointService portainer.EndpointService, endpointIds []portainer.EndpointID) ([]portainer.EndpointType, error) {
+	typeSet := map[portainer.EndpointType]bool{}
+	for _, endpointID := range endpointIds {
+		endpoint, err := endpointService.Endpoint(endpointID)
+		if err != nil {
+			return nil, fmt.Errorf("failed fetching endpoint: %w", err)
+		}
+
+		typeSet[endpoint.Type] = true
+	}
+
+	endpointTypes := make([]portainer.EndpointType, 0, len(typeSet))
+	for endpointType := range typeSet {
+		endpointTypes = append(endpointTypes, endpointType)
+	}
+
+	return endpointTypes, nil
+}

api/http/handler/edgegroups/edgegroup_list_test.go

@@ -0,0 +1,53 @@
+package edgegroups
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_getEndpointTypes(t *testing.T) {
+	endpoints := []portainer.Endpoint{
+		{ID: 1, Type: portainer.DockerEnvironment},
+		{ID: 2, Type: portainer.AgentOnDockerEnvironment},
+		{ID: 3, Type: portainer.AzureEnvironment},
+		{ID: 4, Type: portainer.EdgeAgentOnDockerEnvironment},
+		{ID: 5, Type: portainer.KubernetesLocalEnvironment},
+		{ID: 6, Type: portainer.AgentOnKubernetesEnvironment},
+		{ID: 7, Type: portainer.EdgeAgentOnKubernetesEnvironment},
+	}
+
+	datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints(endpoints))
+
+	tests := []struct {
+		endpointIds []portainer.EndpointID
+		expected    []portainer.EndpointType
+	}{
+		{endpointIds: []portainer.EndpointID{1}, expected: []portainer.EndpointType{portainer.DockerEnvironment}},
+		{endpointIds: []portainer.EndpointID{2}, expected: []portainer.EndpointType{portainer.AgentOnDockerEnvironment}},
+		{endpointIds: []portainer.EndpointID{3}, expected: []portainer.EndpointType{portainer.AzureEnvironment}},
+		{endpointIds: []portainer.EndpointID{4}, expected: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment}},
+		{endpointIds: []portainer.EndpointID{5}, expected: []portainer.EndpointType{portainer.KubernetesLocalEnvironment}},
+		{endpointIds: []portainer.EndpointID{6}, expected: []portainer.EndpointType{portainer.AgentOnKubernetesEnvironment}},
+		{endpointIds: []portainer.EndpointID{7}, expected: []portainer.EndpointType{portainer.EdgeAgentOnKubernetesEnvironment}},
+		{endpointIds: []portainer.EndpointID{7, 2}, expected: []portainer.EndpointType{portainer.EdgeAgentOnKubernetesEnvironment, portainer.AgentOnDockerEnvironment}},
+		{endpointIds: []portainer.EndpointID{6, 4, 1}, expected: []portainer.EndpointType{portainer.AgentOnKubernetesEnvironment, portainer.EdgeAgentOnDockerEnvironment, portainer.DockerEnvironment}},
+		{endpointIds: []portainer.EndpointID{1, 2, 3}, expected: []portainer.EndpointType{portainer.DockerEnvironment, portainer.AgentOnDockerEnvironment, portainer.AzureEnvironment}},
+	}
+
+	for _, test := range tests {
+		ans, err := getEndpointTypes(datastore.Endpoint(), test.endpointIds)
+		assert.NoError(t, err, "getEndpointTypes shouldn't fail")
+
+		assert.ElementsMatch(t, test.expected, ans, "getEndpointTypes expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
+	}
+}
+
+func Test_getEndpointTypes_failWhenEndpointDontExist(t *testing.T) {
+	datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
+
+	_, err := getEndpointTypes(datastore.Endpoint(), []portainer.EndpointID{1})
+	assert.Error(t, err, "getEndpointTypes should fail")
+}

api/http/handler/edgegroups/edgegroup_update.go

@@ -29,7 +29,7 @@ func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("TagIDs is mandatory for a dynamic Edge group")
 	}
 	if !payload.Dynamic && (payload.Endpoints == nil || len(payload.Endpoints) == 0) {
-		return errors.New("Endpoints is mandatory for a static Edge group")
+		return errors.New("Environments is mandatory for a static Edge group")
 	}
 	return nil
 }
@@ -44,7 +44,7 @@ func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
 // @param id path int true "EdgeGroup Id"
 // @param body body edgeGroupUpdatePayload true "EdgeGroup data"
 // @success 200 {object} portainer.EdgeGroup
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @failure 500
 // @router /edge_groups/{id} [put]
 func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -81,12 +81,12 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 	}
 	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments from database", err}
 	}
 
 	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environment groups from database", err}
 	}
 
 	oldRelatedEndpoints := edge.EdgeGroupRelatedEndpoints(edgeGroup, endpoints, endpointGroups)
@@ -99,7 +99,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 		for _, endpointID := range payload.Endpoints {
 			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint from the database", err}
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environment from the database", err}
 			}
 
 			if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
@@ -124,7 +124,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 	for _, endpointID := range endpointsToUpdate {
 		err = handler.updateEndpoint(endpointID)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist Endpoint relation changes inside the database", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist Environment relation changes inside the database", err}
 		}
 	}
 

api/http/handler/edgejobs/edgejob_create.go

@@ -22,10 +22,10 @@ import (
 // @accept json
 // @produce json
 // @param method query string true "Creation Method" Enums(file, string)
-// @param body body edgeJobCreateFromFileContentPayload true "EdgeGroup data when method is string"
-// @param body body edgeJobCreateFromFilePayload true "EdgeGroup data when method is file"
+// @param body_string body edgeJobCreateFromFileContentPayload true "EdgeGroup data when method is string"
+// @param body_file body edgeJobCreateFromFilePayload true "EdgeGroup data when method is file"
 // @success 200 {object} portainer.EdgeGroup
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @failure 500
 // @router /edge_jobs [post]
 func (handler *Handler) edgeJobCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -66,7 +66,7 @@ func (payload *edgeJobCreateFromFileContentPayload) Validate(r *http.Request) er
 	}
 
 	if payload.Endpoints == nil || len(payload.Endpoints) == 0 {
-		return errors.New("Invalid endpoints payload")
+		return errors.New("Invalid environment payload")
 	}
 
 	if govalidator.IsNull(payload.FileContent) {
@@ -119,9 +119,9 @@ func (payload *edgeJobCreateFromFilePayload) Validate(r *http.Request) error {
 	payload.CronExpression = cronExpression
 
 	var endpoints []portainer.EndpointID
-	err = request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, false)
+	err = request.RetrieveMultiPartFormJSONValue(r, "Environments", &endpoints, false)
 	if err != nil {
-		return errors.New("Invalid endpoints")
+		return errors.New("Invalid environments")
 	}
 	payload.Endpoints = endpoints
 
@@ -206,7 +206,7 @@ func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []
 	}
 
 	if len(edgeJob.Endpoints) == 0 {
-		return errors.New("Endpoints are mandatory for an Edge job")
+		return errors.New("Environments are mandatory for an Edge job")
 	}
 
 	scriptPath, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), file)

api/http/handler/edgejobs/edgejob_delete.go

@@ -22,7 +22,7 @@ import (
 // @success 204
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id} [delete]
 func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_file.go

@@ -25,7 +25,7 @@ type edgeJobFileResponse struct {
 // @success 200 {object} edgeJobFileResponse
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id}/file [get]
 func (handler *Handler) edgeJobFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_inspect.go

@@ -26,7 +26,7 @@ type edgeJobInspectResponse struct {
 // @success 200 {object} portainer.EdgeJob
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id} [get]
 func (handler *Handler) edgeJobInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_list.go

@@ -17,7 +17,7 @@ import (
 // @success 200 {array} portainer.EdgeJob
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs [get]
 // GET request on /api/edge_jobs
 func (handler *Handler) edgeJobList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/edgejobs/edgejob_tasklogs_clear.go

@@ -23,7 +23,7 @@ import (
 // @success 204
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id}/tasks/{taskID}/logs [delete]
 func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_tasklogs_collect.go

@@ -22,7 +22,7 @@ import (
 // @success 204
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id}/tasks/{taskID}/logs [post]
 func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_tasklogs_inspect.go

@@ -25,7 +25,7 @@ type fileResponse struct {
 // @success 200 {object} fileResponse
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id}/tasks/{taskID}/logs [get]
 func (handler *Handler) edgeJobTaskLogsInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_tasks_list.go

@@ -28,7 +28,7 @@ type taskContainer struct {
 // @success 200 {array} taskContainer
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id}/tasks [get]
 func (handler *Handler) edgeJobTasksList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_update.go

@@ -40,7 +40,7 @@ func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.EdgeJob
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_jobs/{id} [post]
 func (handler *Handler) edgeJobUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgestacks/edgestack_create.go

@@ -2,6 +2,7 @@ package edgestacks
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -29,7 +30,7 @@ import (
 // @param body_repository body swarmStackFromGitRepositoryPayload true "Required when using method=repository"
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_stacks [post]
 func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	method, err := request.RetrieveQueryParameter(r, "method", false)
@@ -42,37 +43,6 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Edge stack", err}
 	}
 
-	endpoints, err := handler.DataStore.Endpoint().Endpoints()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
-	}
-
-	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
-	}
-
-	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from database", err}
-	}
-
-	relatedEndpoints, err := edge.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, endpoints, endpointGroups, edgeGroups)
-
-	for _, endpointID := range relatedEndpoints {
-		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
-		}
-
-		relation.EdgeStacks[edgeStack.ID] = true
-
-		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
-		}
-	}
-
 	return response.JSON(w, edgeStack)
 }
 
@@ -95,6 +65,11 @@ type swarmStackFromFileContentPayload struct {
 	StackFileContent string `example:"version: 3\n services:\n web:\n image:nginx" validate:"required"`
 	// List of identifiers of EdgeGroups
 	EdgeGroups []portainer.EdgeGroupID `example:"1"`
+	// Deployment type to deploy this stack
+	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
+	// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes endpoints
+	// kubernetes deploytype is enabled only for kubernetes endpoints
+	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
 }
 
 func (payload *swarmStackFromFileContentPayload) Validate(r *http.Request) error {
@@ -124,21 +99,64 @@ func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*porta
 
 	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
-		ID:           portainer.EdgeStackID(stackID),
-		Name:         payload.Name,
-		EntryPoint:   filesystem.ComposeFileDefaultName,
-		CreationDate: time.Now().Unix(),
-		EdgeGroups:   payload.EdgeGroups,
-		Status:       make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		Version:      1,
+		ID:             portainer.EdgeStackID(stackID),
+		Name:           payload.Name,
+		DeploymentType: payload.DeploymentType,
+		CreationDate:   time.Now().Unix(),
+		EdgeGroups:     payload.EdgeGroups,
+		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
+		Version:        1,
+	}
+
+	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
+	if err != nil {
+		return nil, fmt.Errorf("unable to find environment relations in database: %w", err)
+	}
+
+	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
+	if err != nil {
+		return nil, fmt.Errorf("unable to persist environment relation in database: %w", err)
 	}
 
 	stackFolder := strconv.Itoa(int(stack.ID))
-	projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
-	if err != nil {
-		return nil, err
+	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
+		stack.EntryPoint = filesystem.ComposeFileDefaultName
+
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+		if err != nil {
+			return nil, err
+		}
+		stack.ProjectPath = projectPath
+
+		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
+		if err != nil {
+			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
+		}
+
+	} else {
+		hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
+		if err != nil {
+			return nil, fmt.Errorf("unable to check for existence of docker endpoint: %w", err)
+		}
+
+		if hasDockerEndpoint {
+			return nil, fmt.Errorf("edge stack with docker endpoint cannot be deployed with kubernetes config")
+		}
+
+		stack.ManifestPath = filesystem.ManifestFileDefaultName
+
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
+		if err != nil {
+			return nil, err
+		}
+
+		stack.ProjectPath = projectPath
+	}
+
+	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
 	}
-	stack.ProjectPath = projectPath
 
 	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
@@ -162,9 +180,14 @@ type swarmStackFromGitRepositoryPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true.
 	RepositoryPassword string `example:"myGitPassword"`
 	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// List of identifiers of EdgeGroups
 	EdgeGroups []portainer.EdgeGroupID `example:"1"`
+	// Deployment type to deploy this stack
+	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
+	// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes endpoints
+	// kubernetes deploytype is enabled only for kubernetes endpoints
+	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -177,8 +200,8 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
 		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
 	}
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	if govalidator.IsNull(payload.FilePathInRepository) {
+		payload.FilePathInRepository = filesystem.ComposeFileDefaultName
 	}
 	if payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0 {
 		return errors.New("Edge Groups are mandatory for an Edge stack")
@@ -200,32 +223,56 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 
 	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
-		ID:           portainer.EdgeStackID(stackID),
-		Name:         payload.Name,
-		EntryPoint:   payload.ComposeFilePathInRepository,
-		CreationDate: time.Now().Unix(),
-		EdgeGroups:   payload.EdgeGroups,
-		Status:       make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		Version:      1,
+		ID:             portainer.EdgeStackID(stackID),
+		Name:           payload.Name,
+		CreationDate:   time.Now().Unix(),
+		EdgeGroups:     payload.EdgeGroups,
+		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
+		DeploymentType: payload.DeploymentType,
+		Version:        1,
 	}
 
 	projectPath := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
+	if err != nil {
+		return nil, fmt.Errorf("failed fetching relations config: %w", err)
+	}
+
+	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
+	}
+
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
 		return nil, err
 	}
 
+	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
+		stack.EntryPoint = payload.FilePathInRepository
+
+		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
+		if err != nil {
+			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
+		}
+	} else {
+		stack.ManifestPath = payload.FilePathInRepository
+	}
+
+	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
+	}
+
 	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
 		return nil, err
@@ -238,6 +285,7 @@ type swarmStackFromFileUploadPayload struct {
 	Name             string
 	StackFileContent []byte
 	EdgeGroups       []portainer.EdgeGroupID
+	DeploymentType   portainer.EdgeStackDeploymentType
 }
 
 func (payload *swarmStackFromFileUploadPayload) Validate(r *http.Request) error {
@@ -259,6 +307,13 @@ func (payload *swarmStackFromFileUploadPayload) Validate(r *http.Request) error
 		return errors.New("Edge Groups are mandatory for an Edge stack")
 	}
 	payload.EdgeGroups = edgeGroups
+
+	deploymentType, err := request.RetrieveNumericMultiPartFormValue(r, "DeploymentType", true)
+	if err != nil {
+		return errors.New("Invalid deployment type")
+	}
+	payload.DeploymentType = portainer.EdgeStackDeploymentType(deploymentType)
+
 	return nil
 }
 
@@ -276,21 +331,54 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 
 	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
-		ID:           portainer.EdgeStackID(stackID),
-		Name:         payload.Name,
-		EntryPoint:   filesystem.ComposeFileDefaultName,
-		CreationDate: time.Now().Unix(),
-		EdgeGroups:   payload.EdgeGroups,
-		Status:       make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		Version:      1,
+		ID:             portainer.EdgeStackID(stackID),
+		Name:           payload.Name,
+		DeploymentType: payload.DeploymentType,
+		CreationDate:   time.Now().Unix(),
+		EdgeGroups:     payload.EdgeGroups,
+		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
+		Version:        1,
+	}
+
+	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
+	if err != nil {
+		return nil, fmt.Errorf("failed fetching relations config: %w", err)
+	}
+
+	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
 	}
 
 	stackFolder := strconv.Itoa(int(stack.ID))
-	projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
-	if err != nil {
-		return nil, err
+	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
+		stack.EntryPoint = filesystem.ComposeFileDefaultName
+
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+		if err != nil {
+			return nil, err
+		}
+		stack.ProjectPath = projectPath
+
+		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
+		if err != nil {
+			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
+		}
+
+	} else {
+		stack.ManifestPath = filesystem.ManifestFileDefaultName
+
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
+		if err != nil {
+			return nil, err
+		}
+		stack.ProjectPath = projectPath
+	}
+
+	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
 	}
-	stack.ProjectPath = projectPath
 
 	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
@@ -313,3 +401,22 @@ func (handler *Handler) validateUniqueName(name string) error {
 	}
 	return nil
 }
+
+// updateEndpointRelations adds a relation between the Edge Stack to the related endpoints
+func updateEndpointRelations(endpointRelationService portainer.EndpointRelationService, edgeStackID portainer.EdgeStackID, relatedEndpointIds []portainer.EndpointID) error {
+	for _, endpointID := range relatedEndpointIds {
+		relation, err := endpointRelationService.EndpointRelation(endpointID)
+		if err != nil {
+			return fmt.Errorf("unable to find endpoint relation in database: %w", err)
+		}
+
+		relation.EdgeStacks[edgeStackID] = true
+
+		err = endpointRelationService.UpdateEndpointRelation(endpointID, relation)
+		if err != nil {
+			return fmt.Errorf("unable to persist endpoint relation in database: %w", err)
+		}
+	}
+
+	return nil
+}

api/http/handler/edgestacks/edgestack_create_test.go

@@ -0,0 +1,38 @@
+package edgestacks
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_updateEndpointRelation_successfulRuns(t *testing.T) {
+	edgeStackID := portainer.EdgeStackID(5)
+	endpointRelations := []portainer.EndpointRelation{
+		{EndpointID: 1, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+		{EndpointID: 2, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+		{EndpointID: 3, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+		{EndpointID: 4, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+		{EndpointID: 5, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+	}
+
+	relatedIds := []portainer.EndpointID{2, 3}
+
+	dataStore := testhelpers.NewDatastore(testhelpers.WithEndpointRelations(endpointRelations))
+
+	err := updateEndpointRelations(dataStore.EndpointRelation(), edgeStackID, relatedIds)
+
+	assert.NoError(t, err, "updateEndpointRelations should not fail")
+
+	relatedSet := map[portainer.EndpointID]bool{}
+	for _, relationID := range relatedIds {
+		relatedSet[relationID] = true
+	}
+
+	for _, relation := range endpointRelations {
+		shouldBeRelated := relatedSet[relation.EndpointID]
+		assert.Equal(t, shouldBeRelated, relation.EdgeStacks[edgeStackID])
+	}
+}

api/http/handler/edgestacks/edgestack_delete.go

@@ -22,7 +22,7 @@ import (
 // @success 204
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_stacks/{id} [delete]
 func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
@@ -42,34 +42,27 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the edge stack from the database", err}
 	}
 
-	endpoints, err := handler.DataStore.Endpoint().Endpoints()
+	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments relations config from database", err}
 	}
 
-	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
+	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stack related environments from database", err}
 	}
 
-	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from database", err}
-	}
-
-	relatedEndpoints, err := edge.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, endpoints, endpointGroups, edgeGroups)
-
-	for _, endpointID := range relatedEndpoints {
+	for _, endpointID := range relatedEndpointIds {
 		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find environment relation in database", err}
 		}
 
 		delete(relation.EdgeStacks, edgeStack.ID)
 
 		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist environment relation in database", err}
 		}
 	}
 

api/http/handler/edgestacks/edgestack_file.go

@@ -26,7 +26,7 @@ type stackFileResponse struct {
 // @success 200 {object} stackFileResponse
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_stacks/{id}/file [get]
 func (handler *Handler) edgeStackFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
@@ -41,7 +41,12 @@ func (handler *Handler) edgeStackFile(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an edge stack with the specified identifier inside the database", err}
 	}
 
-	stackFileContent, err := handler.FileService.GetFileContent(path.Join(stack.ProjectPath, stack.EntryPoint))
+	fileName := stack.EntryPoint
+	if stack.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
+		fileName = stack.ManifestPath
+	}
+
+	stackFileContent, err := handler.FileService.GetFileContent(path.Join(stack.ProjectPath, fileName))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Compose file from disk", err}
 	}

api/http/handler/edgestacks/edgestack_inspect.go

@@ -21,7 +21,7 @@ import (
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
 // @failure 400
-// @failure 503 Edge compute features are disabled
+// @failure 503 "Edge compute features are disabled"
 // @router /edge_stacks/{id} [get]
 func (handler *Handler) edgeStackInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "id")