fix(buildscripts): make build process more closely resemble EE (#8881)

* fix(editor): fix styles [EE-5369]

* rm hash

---------

Co-authored-by: testa113 <testa113>

.codeclimate.yml

@@ -1,44 +0,0 @@
-version: "2"
-checks:
-  argument-count:
-    enabled: false
-  complex-logic:
-    enabled: false
-  file-lines:
-    enabled: false
-  method-complexity:
-    enabled: false
-  method-count:
-    enabled: false
-  method-lines:
-    enabled: false
-  nested-control-flow:
-    enabled: false
-  return-statements:
-    enabled: false
-  similar-code:
-    enabled: false
-  identical-code:
-    enabled: false
-plugins:
-  gofmt:
-    enabled: true
-  eslint:
-    enabled: true
-    channel: "eslint-5"
-    config:
-      config: .eslintrc.yml
-exclude_patterns:
-- assets/
-- build/
-- dist/
-- distribution/
-- node_modules
-- test/
-- webpack/
-- gruntfile.js
-- webpack.config.js
-- api/
-- "!app/kubernetes/**"
-- .github/
-- .tmp/

.github/workflows/nightly-security-scan.yml

@@ -1,16 +1,16 @@
 name: Nightly Code Security Scan
 
-on: 
+on:
   schedule:
     - cron: '0 8 * * *'
   workflow_dispatch:
-    
+
 jobs:
   client-dependencies:
     name: Client dependency check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
-      github.ref == 'refs/heads/develop' 
+      github.ref == 'refs/heads/develop'
     outputs:
       js: ${{ steps.set-matrix.outputs.js_result }}
     steps:
@@ -24,14 +24,14 @@ jobs:
         with:
           json: true
 
-      - name: Upload js security scan result as artifact 
+      - name: Upload js security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
           path: snyk.json
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
 
       - name: Upload js result html file
@@ -42,7 +42,7 @@ jobs:
 
       - name: Analyse the js result
         id: set-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
           echo "::set-output name=js_result::${result}"
 
@@ -50,7 +50,7 @@ jobs:
     name: Server dependency check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
-      github.ref == 'refs/heads/develop' 
+      github.ref == 'refs/heads/develop'
     outputs:
       go: ${{ steps.set-matrix.outputs.go_result }}
     steps:
@@ -77,8 +77,8 @@ jobs:
           name: go-security-scan-develop-result
           path: snyk.json
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
 
       - name: Upload go result html file
@@ -89,7 +89,7 @@ jobs:
 
       - name: Analyse the go result
         id: set-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
           echo "::set-output name=go_result::${result}"
 
@@ -114,8 +114,11 @@ jobs:
         with:
           node-version: 18.x
 
-      - name: Install packages and build
-        run: yarn install && yarn build
+      - name: Install packages
+        run: yarn --frozen-lockfile
+
+      - name: build
+        run: make build
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -134,9 +137,9 @@ jobs:
 
       - name: Run Trivy vulnerability scanner
         uses: docker://docker.io/aquasec/trivy:latest
-        continue-on-error: true 
+        continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
       - name: Upload image security scan result as artifact
         uses: actions/upload-artifact@v3
@@ -144,8 +147,8 @@ jobs:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
 
       - name: Upload go result html file
@@ -156,7 +159,7 @@ jobs:
 
       - name: Analyse the trivy result
         id: set-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
           echo "::set-output name=image_result::${result}"
 
@@ -167,7 +170,7 @@ jobs:
     if: >-
       github.ref == 'refs/heads/develop'
     strategy:
-      matrix: 
+      matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
         image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
@@ -182,7 +185,7 @@ jobs:
           echo ${{ matrix.image.summary }}
 
       - name: Send Slack message
-        if: >- 
+        if: >-
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
           matrix.image.status == 'failure'

.github/workflows/pr-security.yml

@@ -12,7 +12,7 @@ on:
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
-    
+
 jobs:
   client-dependencies:
     name: Client dependency check
@@ -51,8 +51,8 @@ jobs:
             echo "null" > ./js-snyk-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
 
       - name: Upload js result html file
@@ -63,7 +63,7 @@ jobs:
 
       - name: Analyse the js diff result
         id: set-diff-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
           echo "::set-output name=js_diff_result::${result}"
 
@@ -111,8 +111,8 @@ jobs:
             echo "null" > ./go-snyk-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
 
       - name: Upload go result html file
@@ -123,7 +123,7 @@ jobs:
 
       - name: Analyse the go diff result
         id: set-diff-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
           echo "::set-output name=go_diff_result::${result}"
 
@@ -149,8 +149,11 @@ jobs:
         with:
           node-version: 18.x
 
-      - name: Install packages and build
-        run: yarn install && yarn build
+      - name: Install packages
+        run: yarn --frozen-lockfile
+
+      - name: build
+        run: make build
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -169,9 +172,9 @@ jobs:
 
       - name: Run Trivy vulnerability scanner
         uses: docker://docker.io/aquasec/trivy:latest
-        continue-on-error: true 
+        continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
       - name: Upload image security scan result as artifact
         uses: actions/upload-artifact@v3
@@ -191,8 +194,8 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
+      - name: Export scan result to html file
+        run: |
           $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
 
       - name: Upload image result html file
@@ -203,7 +206,7 @@ jobs:
 
       - name: Analyse the image diff result
         id: set-diff-matrix
-        run: | 
+        run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
           echo "::set-output name=image_diff_result::${result}"
 
@@ -215,17 +218,16 @@ jobs:
       github.event.pull_request &&
       github.event.review.body == '/scan'
     strategy:
-      matrix: 
+      matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
         imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
     steps:
-
       - name: Check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff.status == 'failure' 
+          matrix.imagediff.status == 'failure'
         run: |
           echo ${{ matrix.jsdiff.status }}
           echo ${{ matrix.godiff.status }}

.github/workflows/test.yaml

@@ -8,12 +8,12 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
-        run: yarn test:client
+        run: yarn jest --maxWorkers=2
   # test-server:
   #   runs-on: ubuntu-latest
   #   env:

.github/workflows/validate-openapi-spec.yaml

@@ -0,0 +1,29 @@
+name: Validate OpenAPI specs
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+      - 'release/*'
+
+jobs:
+  openapi-spec:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Download golang modules
+        run: cd ./api && go get -t -v -d ./...
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
+
+      - name: Validate OpenAPI Spec
+        run: make docs-validate

.github/workflows/validate-openapi-spec.yml

@@ -1,53 +0,0 @@
-name: Validate
-
-on:
-  pull_request:
-    branches:
-      - master
-      - develop
-      - 'release/*'
-
-jobs:
-  openapi-spec:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-    - name: Setup Node v14
-      uses: actions/setup-node@v2
-      with:
-        node-version: 14
-
-    # https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - name: Setup Go v1.17.3
-      uses: actions/setup-go@v2
-      with:
-        go-version: '^1.17.3'
-
-    - name: Prebuild docs
-      run: yarn prebuild:docs
-
-    - name: Build OpenAPI 2.0 Spec
-      run: yarn build:docs
-
-    # Install dependencies globally to bypass installing all frontend deps
-    - name: Install swagger2openapi and swagger-cli
-      run: yarn global add swagger2openapi @apidevtools/swagger-cli
-
-    # OpenAPI2.0 does not support multiple body params (which we utilise in some of our handlers).
-    # OAS3.0 however does support multiple body params - hence its best to convert the generated OAS 2.0
-    # to OAS 3.0 and validate the output of generated OAS 3.0 instead.
-    - name: Convert OpenAPI 2.0 to OpenAPI 3.0 and validate spec
-      run: yarn validate:docs

CONTRIBUTING.md

@@ -93,7 +93,7 @@ $ yarn start
 
 Portainer can now be accessed at <https://localhost:9443>.
 
-Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
+Find more detailed steps at <https://docs.portainer.io/contribute/build>.
 
 ### Build customisation
 
@@ -103,6 +103,10 @@ You can customise the following settings:
 - `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`).
 - `PORTAINER_FLAGS`: a list of flags to be used on the portainer commandline, in the form `--admin-password=<pwd hash> --feat fdo=false --feat open-amt` (default: `""`).
 
+## Testing your build
+
+The `--log-level=DEBUG` flag can be passed to the Portainer container in order to provide additional debug output which may be useful when troubleshooting your builds. Please note that this flag was originally intended for internal use and as such the format, functionality and output may change between releases without warning.
+
 ## Adding api docs
 
 When adding a new resource (or a route handler), we should add a new tag to api/http/handler/handler.go#L136 like this:

Makefile

@@ -0,0 +1,122 @@
+# See: https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63
+# For a list of valid GOOS and GOARCH values
+# Note: these can be overriden on the command line e.g. `make PLATFORM=<platform> ARCH=<arch>`
+PLATFORM=$(shell go env GOOS)
+ARCH=$(shell go env GOARCH)
+
+TAG=latest
+SWAG_VERSION=v1.8.11
+
+# build target, can be one of "production", "testing", "development"
+ENV=development
+WEBPACK_CONFIG=webpack/webpack.$(ENV).js
+
+.DEFAULT_GOAL := help
+
+.PHONY: help build-storybook build-client devops download-binaries tidy clean client-deps
+
+##@ Building
+
+init-dist:
+	@mkdir -p dist
+
+build-storybook:
+	yarn storybook:build
+
+build: build-server build-client ## Build the server and client
+
+build-client: init-dist client-deps ## Build the client
+	export NODE_ENV=$(ENV) && yarn build --config $(WEBPACK_CONFIG)
+
+build-server: init-dist ## Build the server binary
+	./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
+
+build-image: build ## Build the Portainer image
+	docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
+
+devops: clean init-dist download-binaries build-client ## Build the server binary for CI
+	echo "Building the devops binary..."
+	@./build/build_binary_azuredevops.sh "$(PLATFORM)" "$(ARCH)"
+
+##@ Dependencies
+
+download-binaries: ## Download dependant binaries
+	@./build/download_binaries.sh $(PLATFORM) $(ARCH)
+
+tidy: ## Tidy up the go.mod file
+	cd api && go mod tidy
+ 
+client-deps: ## Install client dependencies
+	yarn
+
+##@ Cleanup
+
+clean: ## Remove all build and download artifacts
+	@echo "Clearing the dist directory..."
+	@rm -rf dist/*
+
+##@ Testing
+
+test-client: ## Run client tests
+	yarn test
+
+test-server:	## Run server tests
+	cd api && go test -v ./...
+
+test: test-client test-server ## Run all tests
+
+##@ Dev
+
+dev-client: ## Run the client in development mode 
+	yarn dev
+
+dev-server: build-image ## Run the server in development mode
+	@./dev/run_container.sh
+
+
+##@ Format
+
+format-client: ## Format client code
+	yarn format
+
+format-server: ## Format server code
+	cd api && go fmt ./...
+
+format: format-client format-server ## Format all code
+
+##@ Lint
+
+lint: lint-client lint-server ## Lint all code
+
+lint-client: ## Lint client code
+	yarn lint
+
+lint-server: ## Lint server code
+	cd api && go vet ./...
+
+##@ Extension
+
+dev-extension: build-server build-client ## Run the extension in development mode
+	make local -f build/docker-extension/Makefile
+
+##@ Docs
+
+docs-deps: ## Install docs dependencies
+	go install github.com/swaggo/swag/cmd/swag@$(SWAG_VERSION)
+
+docs-build: docs-deps ## Build docs
+	cd api && swag init -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
+
+docs-validate: docs-build ## Validate docs
+	yarn swagger2openapi --warnOnly api/docs/swagger.yaml -o api/docs/openapi.yaml
+	yarn swagger-cli validate api/docs/openapi.yaml
+
+docs-clean: ## Clean docs
+	rm -rf api/docs
+
+docs-validate-clean: docs-validate docs-clean ## Validate and clean docs
+
+##@ Helpers
+
+help:  ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

api/backup/restore.go

@@ -3,8 +3,10 @@ package backup
 import (
 	"context"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/pkg/errors"
@@ -43,6 +45,12 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
 		return errors.Wrap(err, "Failed to stop db")
 	}
 
+	// At some point, backups were created containing a subdirectory, now we need to handle both
+	restorePath, err = getRestoreSourcePath(restorePath)
+	if err != nil {
+		return errors.Wrap(err, "failed to restore from backup. Portainer database missing from backup file")
+	}
+
 	if err = restoreFiles(restorePath, filestorePath); err != nil {
 		return errors.Wrap(err, "failed to restore the system state")
 	}
@@ -59,6 +67,26 @@ func extractArchive(r io.Reader, destinationDirPath string) error {
 	return archive.ExtractTarGz(r, destinationDirPath)
 }
 
+func getRestoreSourcePath(dir string) (string, error) {
+	// find portainer.db or portainer.edb file. Return the parent directory
+	var portainerdbRegex = regexp.MustCompile(`^portainer.e?db$`)
+
+	backupDirPath := dir
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if portainerdbRegex.MatchString(d.Name()) {
+			backupDirPath = filepath.Dir(path)
+			return filepath.SkipDir
+		}
+		return nil
+	})
+
+	return backupDirPath, err
+}
+
 func restoreFiles(srcDir string, destinationDir string) error {
 	for _, filename := range filesToRestore {
 		err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)

api/cmd/portainer/main.go

@@ -689,20 +689,16 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing upgrade service")
 	}
 
-	// FIXME: In 2.16 we changed the way ingress controller permissions are
-	// stored. Instead of being stored as annotation on an ingress rule, we keep
-	// them in our database. However, in order to run the migration we need an
-	// admin kube client to run lookup the old ingress rules and compare them
-	// with the current existing ingress classes.
-	//
-	// Unfortunately, our migrations run as part of the database initialization
-	// and our kubeclients require an initialized database. So it is not
-	// possible to do this migration as part of our normal flow. We DO have a
-	// migration which toggles a boolean in kubernetes configuration that
-	// indicated that this "post init" migration should be run. If/when this is
-	// resolved we can remove this function.
-	err = kubernetesClientFactory.PostInitMigrateIngresses()
-	if err != nil {
+	// Our normal migrations run as part of the database initialization
+	// but some more complex migrations require access to a kubernetes or docker
+	// client. Therefore we run a separate migration process just before
+	// starting the server.
+	postInitMigrator := datastore.NewPostInitMigrator(
+		kubernetesClientFactory,
+		dockerClientFactory,
+		dataStore,
+	)
+	if err := postInitMigrator.PostInitMigrate(); err != nil {
 		log.Fatal().Err(err).Msg("failure during post init migrations")
 	}
 

api/dataservices/stack/tests/stack_test.go

@@ -68,13 +68,13 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 
 	if webhookID == "" {
 		if b.count%2 == 0 {
-			stack.AutoUpdate = &portainer.StackAutoUpdate{
+			stack.AutoUpdate = &portainer.AutoUpdateSettings{
 				Interval: "",
 				Webhook:  "",
 			}
 		} // else keep AutoUpdate nil
 	} else {
-		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
+		stack.AutoUpdate = &portainer.AutoUpdateSettings{Webhook: webhookID}
 	}
 
 	err := b.store.StackService.Create(&stack)
@@ -91,8 +91,8 @@ func Test_RefreshableStacks(t *testing.T) {
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}
-	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
-	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
+	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}
+	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
 
 	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
 		err := store.Stack().Create(stack)

api/datastore/migrate_post_init.go

@@ -0,0 +1,116 @@
+package datastore
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/docker"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/rs/zerolog/log"
+)
+
+type PostInitMigrator struct {
+	kubeFactory   *cli.ClientFactory
+	dockerFactory *docker.ClientFactory
+	dataStore     dataservices.DataStore
+}
+
+func NewPostInitMigrator(
+	kubeFactory *cli.ClientFactory,
+	dockerFactory *docker.ClientFactory,
+	dataStore dataservices.DataStore,
+) *PostInitMigrator {
+	return &PostInitMigrator{
+		kubeFactory:   kubeFactory,
+		dockerFactory: dockerFactory,
+		dataStore:     dataStore,
+	}
+}
+
+func (migrator *PostInitMigrator) PostInitMigrate() error {
+	if err := migrator.PostInitMigrateIngresses(); err != nil {
+		return err
+	}
+
+	migrator.PostInitMigrateGPUs()
+
+	return nil
+}
+
+func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
+	endpoints, err := migrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		return err
+	}
+	for i := range endpoints {
+		// Early exit if we do not need to migrate!
+		if endpoints[i].PostInitMigrations.MigrateIngresses == false {
+			return nil
+		}
+
+		err := migrator.kubeFactory.MigrateEndpointIngresses(&endpoints[i])
+		if err != nil {
+			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
+		}
+	}
+
+	return nil
+}
+
+// PostInitMigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
+// If there's an error getting the containers, we'll log it and move on
+func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
+	environments, err := migrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		log.Err(err).Msg("failure getting endpoints")
+		return
+	}
+	for i := range environments {
+		if environments[i].Type == portainer.DockerEnvironment {
+			// // Early exit if we do not need to migrate!
+			if environments[i].PostInitMigrations.MigrateGPUs == false {
+				return
+			}
+
+			// set the MigrateGPUs flag to false so we don't run this again
+			environments[i].PostInitMigrations.MigrateGPUs = false
+			migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
+
+			// create a docker client
+			dockerClient, err := migrator.dockerFactory.CreateClient(&environments[i], "", nil)
+			if err != nil {
+				log.Err(err).Msg("failure creating docker client for environment: " + environments[i].Name)
+				return
+			}
+			defer dockerClient.Close()
+
+			// get all containers
+			containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+			if err != nil {
+				log.Err(err).Msg("failed to list containers")
+				return
+			}
+
+			// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole endpoint
+		containersLoop:
+			for _, container := range containers {
+				// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
+				containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
+				if err != nil {
+					log.Err(err).Msg("failed to inspect container")
+					return
+				}
+				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
+				for _, deviceRequest := range deviceRequests {
+					if deviceRequest.Driver == "nvidia" {
+						environments[i].EnableGPUManagement = true
+						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
+						break containersLoop
+					}
+				}
+			}
+		}
+	}
+}

api/datastore/migrator/migrate_dbversion90.go

@@ -3,11 +3,16 @@ package migrator
 import (
 	"github.com/rs/zerolog/log"
 
+	portainer "github.com/portainer/portainer/api"
 	portainerDsErrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
 func (m *Migrator) migrateDBVersionToDB90() error {
-	if err := m.updateUserThemForDB90(); err != nil {
+	if err := m.updateUserThemeForDB90(); err != nil {
+		return err
+	}
+
+	if err := m.updateEnableGpuManagementFeatures(); err != nil {
 		return err
 	}
 
@@ -39,7 +44,7 @@ func (m *Migrator) updateEdgeStackStatusForDB90() error {
 	return nil
 }
 
-func (m *Migrator) updateUserThemForDB90() error {
+func (m *Migrator) updateUserThemeForDB90() error {
 	log.Info().Msg("updating existing user theme settings")
 
 	users, err := m.userService.Users()
@@ -60,3 +65,28 @@ func (m *Migrator) updateUserThemForDB90() error {
 
 	return nil
 }
+
+func (m *Migrator) updateEnableGpuManagementFeatures() error {
+	// get all environments
+	environments, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, environment := range environments {
+		if environment.Type == portainer.DockerEnvironment {
+			// set the PostInitMigrations.MigrateGPUs to true on this environment to run the migration only on the 2.18 upgrade
+			environment.PostInitMigrations.MigrateGPUs = true
+			// if there's one or more gpu, set the EnableGpuManagement setting to true
+			gpuList := environment.Gpus
+			if len(gpuList) > 0 {
+				environment.EnableGPUManagement = true
+			}
+			// update the environment
+			if err := m.endpointService.UpdateEndpoint(environment.ID, &environment); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

api/datastore/test_data/input_24.json

@@ -37,6 +37,7 @@
       "EdgeKey": "",
       "Extensions": [],
       "GroupId": 1,
+      "Heartbeat": false,
       "Id": 1,
       "Name": "local",
       "PublicURL": "",

api/datastore/test_data/output_24_to_latest.json

@@ -46,8 +46,10 @@
       },
       "EdgeCheckinInterval": 0,
       "EdgeKey": "",
+      "EnableGPUManagement": false,
       "Gpus": [],
       "GroupId": 1,
+      "Heartbeat": false,
       "Id": 1,
       "IsEdgeDevice": false,
       "Kubernetes": {
@@ -63,6 +65,7 @@
           "UseServerMetrics": false
         },
         "Flags": {
+          "IsServerIngressClassDetected": false,
           "IsServerMetricsDetected": false,
           "IsServerStorageDetected": false
         },
@@ -71,6 +74,7 @@
       "LastCheckInDate": 0,
       "Name": "local",
       "PostInitMigrations": {
+        "MigrateGPUs": true,
         "MigrateIngresses": true
       },
       "PublicURL": "",
@@ -903,8 +907,7 @@
       },
       "Role": 1,
       "ThemeSettings": {
-        "color": "",
-        "subtleUpgradeButton": false
+        "color": ""
       },
       "TokenIssueAt": 0,
       "UserTheme": "",
@@ -934,8 +937,7 @@
       },
       "Role": 1,
       "ThemeSettings": {
-        "color": "",
-        "subtleUpgradeButton": false
+        "color": ""
       },
       "TokenIssueAt": 0,
       "UserTheme": "",
@@ -943,6 +945,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.18.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.18.2\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/snapshot.go

@@ -220,7 +220,9 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 	snapshot.HealthyContainerCount = healthyContainers
 	snapshot.UnhealthyContainerCount = unhealthyContainers
 	snapshot.StackCount += len(stacks)
-	snapshot.SnapshotRaw.Containers = containers
+	for _, container := range containers {
+		snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
+	}
 	return nil
 }
 

api/exec/compose_stack.go

@@ -53,7 +53,7 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack, false)
+	filePaths := stackutils.GetStackFilePaths(stack, true)
 	err = manager.deployer.Deploy(ctx, filePaths, libstack.DeployOptions{
 		Options: libstack.Options{
 			WorkingDir:  stack.ProjectPath,
@@ -66,7 +66,7 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 	return errors.Wrap(err, "failed to deploy a stack")
 }
 
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+// Down stops and removes containers, networks, images, and volumes
 func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := manager.fetchEndpointProxy(endpoint)
 	if err != nil {
@@ -81,14 +81,12 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack, false)
-
-	err = manager.deployer.Remove(ctx, filePaths, libstack.Options{
+	err = manager.deployer.Remove(ctx, stack.Name, nil, libstack.Options{
 		WorkingDir:  stack.ProjectPath,
 		EnvFilePath: envFilePath,
 		Host:        url,
-		ProjectName: stack.Name,
 	})
+
 	return errors.Wrap(err, "failed to remove a stack")
 }
 
@@ -108,7 +106,7 @@ func (manager *ComposeStackManager) Pull(ctx context.Context, stack *portainer.S
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack, false)
+	filePaths := stackutils.GetStackFilePaths(stack, true)
 	err = manager.deployer.Pull(ctx, filePaths, libstack.Options{
 		WorkingDir:  stack.ProjectPath,
 		EnvFilePath: envFilePath,

api/exec/swarm_stack.go

@@ -90,7 +90,7 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, pullImage bool, endpoint *portainer.Endpoint) error {
-	filePaths := stackutils.GetStackFilePaths(stack, false)
+	filePaths := stackutils.GetStackFilePaths(stack, true)
 	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	if err != nil {
 		return err

api/git/azure.go

@@ -15,8 +15,6 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 	gittypes "github.com/portainer/portainer/api/git/types"
 
-	"github.com/go-git/go-git/v5/plumbing/transport/client"
-	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
 )
 
@@ -51,21 +49,21 @@ type azureItem struct {
 }
 
 type azureClient struct {
-	client  *http.Client
 	baseUrl string
 }
 
 func NewAzureClient() *azureClient {
-	httpsCli := newHttpClientForAzure()
 	return &azureClient{
-		client:  httpsCli,
 		baseUrl: "https://dev.azure.com",
 	}
 }
 
-func newHttpClientForAzure() *http.Client {
+func newHttpClientForAzure(insecureSkipVerify bool) *http.Client {
 	tlsConfig := crypto.CreateTLSConfiguration()
-	tlsConfig.InsecureSkipVerify = true
+
+	if insecureSkipVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
 
 	httpsCli := &http.Client{
 		Transport: &http.Transport{
@@ -75,7 +73,6 @@ func newHttpClientForAzure() *http.Client {
 		Timeout: 300 * time.Second,
 	}
 
-	client.InstallProtocol("https", githttp.NewClient(httpsCli))
 	return httpsCli
 }
 
@@ -107,6 +104,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to create temp file")
 	}
+
 	defer zipFile.Close()
 
 	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
@@ -120,10 +118,14 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
 		return "", errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
-	res, err := a.client.Do(req)
+	client := newHttpClientForAzure(opt.tlsSkipVerify)
+	defer client.CloseIdleConnections()
+
+	res, err := client.Do(req)
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to make an HTTP request")
 	}
+
 	defer res.Body.Close()
 
 	if res.StatusCode != http.StatusOK {
@@ -167,7 +169,10 @@ func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureI
 		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
-	resp, err := a.client.Do(req)
+	client := newHttpClientForAzure(opt.tlsSkipVerify)
+	defer client.CloseIdleConnections()
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to make an HTTP request")
 	}
@@ -400,7 +405,10 @@ func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, e
 		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
-	resp, err := a.client.Do(req)
+	client := newHttpClientForAzure(opt.tlsSkipVerify)
+	defer client.CloseIdleConnections()
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to make an HTTP request")
 	}
@@ -457,7 +465,10 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
 		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
-	resp, err := a.client.Do(req)
+	client := newHttpClientForAzure(opt.tlsSkipVerify)
+	defer client.CloseIdleConnections()
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to make an HTTP request")
 	}

api/git/azure_integration_test.go

@@ -59,7 +59,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			dst := t.TempDir()
 			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
-			err := service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "")
+			err := service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "", false)
 			assert.NoError(t, err)
 			assert.FileExists(t, filepath.Join(dst, "README.md"))
 		})
@@ -74,7 +74,7 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 
 	dst := t.TempDir()
 
-	err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat)
+	err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat, false)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
@@ -85,7 +85,7 @@ func TestService_LatestCommitID_Azure(t *testing.T) {
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
 	service := NewService(context.TODO())
 
-	id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat)
+	id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat, false)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
@@ -97,7 +97,7 @@ func TestService_ListRefs_Azure(t *testing.T) {
 	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
 	service := NewService(context.TODO())
 
-	refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(refs), 1)
 }
@@ -109,8 +109,8 @@ func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
 	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
 	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
 
-	go service.ListRefs(privateAzureRepoURL, username, accessToken, false)
-	service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	go service.ListRefs(privateAzureRepoURL, username, accessToken, false, false)
+	service.ListRefs(privateAzureRepoURL, username, accessToken, false, false)
 
 	time.Sleep(2 * time.Second)
 }
@@ -248,7 +248,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
 			if tt.expect.shouldFail {
 				assert.Error(t, err)
 				if tt.expect.err != nil {
@@ -271,8 +271,8 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
 	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
 	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
 
-	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
-	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
+	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
 
 	time.Sleep(2 * time.Second)
 }

api/git/azure_test.go

@@ -292,7 +292,6 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 			defer server.Close()
 
 			a := &azureClient{
-				client:  server.Client(),
 				baseUrl: server.URL,
 			}
 
@@ -329,7 +328,6 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
 	defer server.Close()
 
 	a := &azureClient{
-		client:  server.Client(),
 		baseUrl: server.URL,
 	}
 
@@ -442,6 +440,7 @@ func Test_listRefs_azure(t *testing.T) {
 
 	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
 	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+
 	tests := []struct {
 		name   string
 		args   baseOption

api/git/backup.go

@@ -0,0 +1,64 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	ErrInvalidGitCredential = errors.New("Invalid git credential")
+)
+
+type CloneOptions struct {
+	ProjectPath   string
+	URL           string
+	ReferenceName string
+	Username      string
+	Password      string
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
+}
+
+func CloneWithBackup(gitService portainer.GitService, fileService portainer.FileService, options CloneOptions) (clean func(), err error) {
+	backupProjectPath := fmt.Sprintf("%s-old", options.ProjectPath)
+	cleanUp := false
+	cleanFn := func() {
+		if !cleanUp {
+			return
+		}
+
+		err = fileService.RemoveDirectory(backupProjectPath)
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to remove git repository directory")
+		}
+	}
+
+	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath)
+	if err != nil {
+		return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
+	}
+
+	cleanUp = true
+
+	err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password, options.TLSSkipVerify)
+	if err != nil {
+		cleanUp = false
+		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath)
+		if restoreError != nil {
+			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
+		}
+
+		if err == gittypes.ErrAuthenticationFailure {
+			return cleanFn, errors.WithMessage(err, ErrInvalidGitCredential.Error())
+		}
+
+		return cleanFn, errors.WithMessage(err, "Unable to clone git repository")
+	}
+
+	return cleanFn, nil
+}

api/git/credentials.go

@@ -0,0 +1,13 @@
+package git
+
+import (
+	gittypes "github.com/portainer/portainer/api/git/types"
+)
+
+func GetCredentials(auth *gittypes.GitAuthentication) (string, string, error) {
+	if auth == nil {
+		return "", "", nil
+	}
+
+	return auth.Username, auth.Password, nil
+}

api/git/git.go

@@ -28,9 +28,10 @@ func NewGitClient(preserveGitDir bool) *gitClient {
 
 func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error {
 	gitOptions := git.CloneOptions{
-		URL:   opt.repositoryUrl,
-		Depth: opt.depth,
-		Auth:  getAuth(opt.username, opt.password),
+		URL:             opt.repositoryUrl,
+		Depth:           opt.depth,
+		InsecureSkipTLS: opt.tlsSkipVerify,
+		Auth:            getAuth(opt.username, opt.password),
 	}
 
 	if opt.referenceName != "" {
@@ -60,7 +61,8 @@ func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string
 	})
 
 	listOptions := &git.ListOptions{
-		Auth: getAuth(opt.username, opt.password),
+		Auth:            getAuth(opt.username, opt.password),
+		InsecureSkipTLS: opt.tlsSkipVerify,
 	}
 
 	refs, err := remote.List(listOptions)
@@ -110,7 +112,8 @@ func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, err
 	})
 
 	listOptions := &git.ListOptions{
-		Auth: getAuth(opt.username, opt.password),
+		Auth:            getAuth(opt.username, opt.password),
+		InsecureSkipTLS: opt.tlsSkipVerify,
 	}
 
 	refs, err := rem.List(listOptions)
@@ -132,12 +135,13 @@ func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, err
 // listFiles list all filenames under the specific repository
 func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
 	cloneOption := &git.CloneOptions{
-		URL:           opt.repositoryUrl,
-		NoCheckout:    true,
-		Depth:         1,
-		SingleBranch:  true,
-		ReferenceName: plumbing.ReferenceName(opt.referenceName),
-		Auth:          getAuth(opt.username, opt.password),
+		URL:             opt.repositoryUrl,
+		NoCheckout:      true,
+		Depth:           1,
+		SingleBranch:    true,
+		ReferenceName:   plumbing.ReferenceName(opt.referenceName),
+		Auth:            getAuth(opt.username, opt.password),
+		InsecureSkipTLS: opt.tlsSkipVerify,
 	}
 
 	repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)

api/git/git_integration_test.go

@@ -24,7 +24,7 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	dst := t.TempDir()
 
 	repositoryUrl := privateGitRepoURL
-	err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
+	err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken, false)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
@@ -37,7 +37,7 @@ func TestService_LatestCommitID_GitHub(t *testing.T) {
 	service := newService(context.TODO(), 0, 0)
 
 	repositoryUrl := privateGitRepoURL
-	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken, false)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
@@ -50,7 +50,7 @@ func TestService_ListRefs_GitHub(t *testing.T) {
 	service := newService(context.TODO(), 0, 0)
 
 	repositoryUrl := privateGitRepoURL
-	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(refs), 1)
 }
@@ -63,8 +63,8 @@ func TestService_ListRefs_Github_Concurrently(t *testing.T) {
 	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
 
 	repositoryUrl := privateGitRepoURL
-	go service.ListRefs(repositoryUrl, username, accessToken, false)
-	service.ListRefs(repositoryUrl, username, accessToken, false)
+	go service.ListRefs(repositoryUrl, username, accessToken, false, false)
+	service.ListRefs(repositoryUrl, username, accessToken, false, false)
 
 	time.Sleep(2 * time.Second)
 }
@@ -202,7 +202,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
 			if tt.expect.shouldFail {
 				assert.Error(t, err)
 				if tt.expect.err != nil {
@@ -226,8 +226,8 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
 
-	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
 
 	time.Sleep(2 * time.Second)
 }
@@ -240,8 +240,8 @@ func TestService_purgeCache_Github(t *testing.T) {
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := NewService(context.TODO())
 
-	service.ListRefs(repositoryUrl, username, accessToken, false)
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListRefs(repositoryUrl, username, accessToken, false, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
 
 	assert.Equal(t, 1, service.repoRefCache.Len())
 	assert.Equal(t, 1, service.repoFileCache.Len())
@@ -261,8 +261,8 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) {
 	// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
 	service := newService(context.TODO(), 2, 40*timeout)
 
-	service.ListRefs(repositoryUrl, username, accessToken, false)
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListRefs(repositoryUrl, username, accessToken, false, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
@@ -293,12 +293,12 @@ func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
 	service := newService(context.TODO(), 2, 0)
 
 	repositoryUrl := privateGitRepoURL
-	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(refs), 1)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	_, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false)
 	assert.Error(t, err)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 }
@@ -311,26 +311,26 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
 	service := newService(context.TODO(), 2, 0)
 
 	repositoryUrl := privateGitRepoURL
-	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(refs), 1)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
-	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{})
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 2, service.repoFileCache.Len())
 
-	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	_, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false)
 	assert.Error(t, err)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	_, err = service.ListRefs(repositoryUrl, username, "fake-token", true, false)
 	assert.Error(t, err)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 	// The relevant file caches should be removed too
@@ -344,12 +344,12 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	repositoryUrl := privateGitRepoURL
-	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
-	files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{})
+	_, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}, false)
 	assert.Error(t, err)
 	assert.Equal(t, 0, service.repoFileCache.Len())
 }

api/git/git_test.go

@@ -38,7 +38,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) {
 
 	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
-	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
 }
@@ -50,7 +50,7 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
 
 	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
-	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false)
 	assert.NoError(t, err)
 	assert.NoDirExists(t, filepath.Join(dir, ".git"))
 }
@@ -84,7 +84,7 @@ func Test_latestCommitID(t *testing.T) {
 	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
 
-	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "")
+	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "", false)
 
 	assert.NoError(t, err)
 	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)

api/git/service.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -20,6 +21,7 @@ type baseOption struct {
 	repositoryUrl string
 	username      string
 	password      string
+	tlsSkipVerify bool
 }
 
 // fetchOption allows to specify the reference name of the target repository
@@ -119,13 +121,14 @@ func (service *Service) timerHasStopped() bool {
 
 // CloneRepository clones a git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error {
 	options := cloneOption{
 		fetchOption: fetchOption{
 			baseOption: baseOption{
 				repositoryUrl: repositoryURL,
 				username:      username,
 				password:      password,
+				tlsSkipVerify: tlsSkipVerify,
 			},
 			referenceName: referenceName,
 		},
@@ -144,12 +147,13 @@ func (service *Service) cloneRepository(destination string, options cloneOption)
 }
 
 // LatestCommitID returns SHA1 of the latest commit of the specified reference
-func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
 	options := fetchOption{
 		baseOption: baseOption{
 			repositoryUrl: repositoryURL,
 			username:      username,
 			password:      password,
+			tlsSkipVerify: tlsSkipVerify,
 		},
 		referenceName: referenceName,
 	}
@@ -162,8 +166,8 @@ func (service *Service) LatestCommitID(repositoryURL, referenceName, username, p
 }
 
 // ListRefs will list target repository's references without cloning the repository
-func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
-	refCacheKey := generateCacheKey(repositoryURL, password)
+func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
+	refCacheKey := generateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify))
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
 		service.repoRefCache.Remove(refCacheKey)
@@ -193,6 +197,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 		repositoryUrl: repositoryURL,
 		username:      username,
 		password:      password,
+		tlsSkipVerify: tlsSkipVerify,
 	}
 
 	var (
@@ -219,8 +224,8 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 
 // ListFiles will list all the files of the target repository with specific extensions.
 // If extension is not provided, it will list all the files under the target repository
-func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
-	repoKey := generateCacheKey(repositoryURL, referenceName)
+func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
+	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify))
 
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
@@ -246,6 +251,7 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 			repositoryUrl: repositoryURL,
 			username:      username,
 			password:      password,
+			tlsSkipVerify: tlsSkipVerify,
 		},
 		referenceName: referenceName,
 	}

api/git/types/types.go

@@ -3,8 +3,8 @@ package gittypes
 import "errors"
 
 var (
-	ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
-	ErrAuthenticationFailure  = errors.New("Authentication failed, please ensure that the git credentials are correct.")
+	ErrIncorrectRepositoryURL = errors.New("git repository could not be found, please ensure that the URL is correct")
+	ErrAuthenticationFailure  = errors.New("authentication failed, please ensure that the git credentials are correct")
 )
 
 // RepoConfig represents a configuration for a repo
@@ -19,6 +19,8 @@ type RepoConfig struct {
 	Authentication *GitAuthentication
 	// Repository hash
 	ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
 type GitAuthentication struct {

api/git/update/update.go

@@ -0,0 +1,96 @@
+package update
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/git"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/rs/zerolog/log"
+)
+
+// UpdateGitObject updates a git object based on its config
+func UpdateGitObject(gitService portainer.GitService, objId string, gitConfig *gittypes.RepoConfig, forceUpdate bool, projectPath string) (bool, string, error) {
+	if gitConfig == nil {
+		return false, "", nil
+	}
+
+	log.Debug().
+		Str("url", gitConfig.URL).
+		Str("ref", gitConfig.ReferenceName).
+		Str("object", objId).
+		Msg("the object has a git config, try to poll from git repository")
+
+	username, password, err := git.GetCredentials(gitConfig.Authentication)
+	if err != nil {
+		return false, "", errors.WithMessagef(err, "failed to get credentials for %v", objId)
+	}
+
+	newHash, err := gitService.LatestCommitID(gitConfig.URL, gitConfig.ReferenceName, username, password, gitConfig.TLSSkipVerify)
+	if err != nil {
+		return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId)
+	}
+
+	hashChanged := !strings.EqualFold(newHash, gitConfig.ConfigHash)
+
+	if !hashChanged && !forceUpdate {
+		log.Debug().
+			Str("hash", newHash).
+			Str("url", gitConfig.URL).
+			Str("ref", gitConfig.ReferenceName).
+			Str("object", objId).
+			Msg("git repo is up to date")
+
+		return false, newHash, nil
+	}
+
+	cloneParams := &cloneRepositoryParameters{
+		url:           gitConfig.URL,
+		ref:           gitConfig.ReferenceName,
+		toDir:         projectPath,
+		tlsSkipVerify: gitConfig.TLSSkipVerify,
+	}
+	if gitConfig.Authentication != nil {
+		cloneParams.auth = &gitAuth{
+			username: username,
+			password: password,
+		}
+	}
+
+	if err := cloneGitRepository(gitService, cloneParams); err != nil {
+		return false, "", errors.WithMessagef(err, "failed to do a fresh clone of %v", objId)
+	}
+
+	log.Debug().
+		Str("hash", newHash).
+		Str("url", gitConfig.URL).
+		Str("ref", gitConfig.ReferenceName).
+		Str("object", objId).
+		Msg("git repo cloned updated")
+
+	return true, newHash, nil
+}
+
+type cloneRepositoryParameters struct {
+	url   string
+	ref   string
+	toDir string
+	auth  *gitAuth
+	// tlsSkipVerify skips SSL verification when cloning the Git repository
+	tlsSkipVerify bool `example:"false"`
+}
+
+type gitAuth struct {
+	username string
+	password string
+}
+
+func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error {
+	if cloneParams.auth != nil {
+		return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password, cloneParams.tlsSkipVerify)
+	}
+
+	return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "", cloneParams.tlsSkipVerify)
+}

api/git/update/validate.go

@@ -0,0 +1,31 @@
+package update
+
+import (
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	portainer "github.com/portainer/portainer/api"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+)
+
+func ValidateAutoUpdateSettings(autoUpdate *portainer.AutoUpdateSettings) error {
+	if autoUpdate == nil {
+		return nil
+	}
+
+	if autoUpdate.Webhook == "" && autoUpdate.Interval == "" {
+		return httperrors.NewInvalidPayloadError("Webhook or Interval must be provided")
+	}
+
+	if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) {
+		return httperrors.NewInvalidPayloadError("invalid Webhook format")
+	}
+
+	if autoUpdate.Interval != "" {
+		if _, err := time.ParseDuration(autoUpdate.Interval); err != nil {
+			return httperrors.NewInvalidPayloadError("invalid Interval format")
+		}
+	}
+
+	return nil
+}

api/git/update/validate_test.go

@@ -1,4 +1,4 @@
-package stackutils
+package update
 
 import (
 	"testing"
@@ -7,25 +7,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_ValidateStackAutoUpdate(t *testing.T) {
+func Test_ValidateAutoUpdate(t *testing.T) {
 	tests := []struct {
 		name    string
-		value   *portainer.StackAutoUpdate
+		value   *portainer.AutoUpdateSettings
 		wantErr bool
 	}{
 		{
 			name:    "webhook is not a valid UUID",
-			value:   &portainer.StackAutoUpdate{Webhook: "fake-webhook"},
+			value:   &portainer.AutoUpdateSettings{Webhook: "fake-webhook"},
 			wantErr: true,
 		},
 		{
 			name:    "incorrect interval value",
-			value:   &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"},
+			value:   &portainer.AutoUpdateSettings{Interval: "1dd2hh3mm"},
 			wantErr: true,
 		},
 		{
 			name: "valid auto update",
-			value: &portainer.StackAutoUpdate{
+			value: &portainer.AutoUpdateSettings{
 				Webhook:  "8dce8c2f-9ca1-482b-ad20-271e86536ada",
 				Interval: "5h30m40s10ms",
 			},
@@ -35,7 +35,7 @@ func Test_ValidateStackAutoUpdate(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := ValidateStackAutoUpdate(tt.value)
+			err := ValidateAutoUpdateSettings(tt.value)
 			assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err)
 		})
 	}

api/git/validate.go

@@ -0,0 +1,25 @@
+package git
+
+import (
+	"github.com/asaskevich/govalidator"
+
+	gittypes "github.com/portainer/portainer/api/git/types"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+)
+
+func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
+	if govalidator.IsNull(repoConfig.URL) || !govalidator.IsURL(repoConfig.URL) {
+		return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
+	}
+
+	return ValidateRepoAuthentication(repoConfig.Authentication)
+
+}
+
+func ValidateRepoAuthentication(auth *gittypes.GitAuthentication) error {
+	if auth != nil && govalidator.IsNull(auth.Password) {
+		return httperrors.NewInvalidPayloadError("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+
+	return nil
+}

api/go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27
+	github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7
 	github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a
 	github.com/portainer/libhttp v0.0.0-20230206214615-dabd58de9f44
 	github.com/portainer/portainer/pkg/featureflags v0.0.0-20230209201943-d73622ed9cd4

api/go.sum

@@ -342,8 +342,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27 h1:PceCpp86SDYb3lZHT4KpuBCkmcJMW5x1qrdFNEfAdUo=
-github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7 h1:/i985KPNw0KvVtLhTEPUa86aJMtun5ZPOyFCJzdY+dY=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a h1:B0z3skIMT+OwVNJPQhKp52X+9OWW6A9n5UWig3lHBJk=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
 github.com/portainer/libhttp v0.0.0-20230206214615-dabd58de9f44 h1:4LYprPd3TsYjHk7CaTmCov1ceG6VKJsL40fJIWiRxpw=

api/http/errors/invalidpayload.go

@@ -0,0 +1,13 @@
+package errors
+
+type InvalidPayloadError struct {
+	msg string
+}
+
+func (e *InvalidPayloadError) Error() string {
+	return e.msg
+}
+
+func NewInvalidPayloadError(msg string) *InvalidPayloadError {
+	return &InvalidPayloadError{msg: msg}
+}

api/http/handler/backup/handler.go

@@ -61,10 +61,12 @@ func adminAccess(next http.Handler) http.Handler {
 		securityContext, err := security.RetrieveRestrictedRequestContext(r)
 		if err != nil {
 			httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user info from request context", err)
+			return
 		}
 
 		if !securityContext.IsAdmin {
 			httperror.WriteError(w, http.StatusUnauthorized, "User is not authorized to perform the action", nil)
+			return
 		}
 
 		next.ServeHTTP(w, r)

api/http/handler/customtemplates/customtemplate_create.go

@@ -213,6 +213,8 @@ type customTemplateFromGitRepositoryPayload struct {
 	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -279,7 +281,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		repositoryPassword = ""
 	}
 
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword, payload.TLSSkipVerify)
 	if err != nil {
 		if err == gittypes.ErrAuthenticationFailure {
 			return nil, fmt.Errorf("invalid git credential")

api/http/handler/customtemplates/customtemplate_delete.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/rs/zerolog/log"
 )
 
 // @id CustomTemplateDelete
@@ -61,7 +62,7 @@ func (handler *Handler) customTemplateDelete(w http.ResponseWriter, r *http.Requ
 
 	err = handler.FileService.RemoveDirectory(customTemplate.ProjectPath)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove custom template files from disk", err)
+		log.Warn().Err(err).Msg("Unable to remove custom template files from disk")
 	}
 
 	if resourceControl != nil {

api/http/handler/edgejobs/edgejob_delete.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/maps"
+	"github.com/rs/zerolog/log"
 )
 
 // @id EdgeJobDelete
@@ -18,7 +19,7 @@ import (
 // @tags edge_jobs
 // @security ApiKeyAuth
 // @security jwt
-// @param id path string true "EdgeJob Id"
+// @param id path int true "EdgeJob Id"
 // @success 204
 // @failure 500
 // @failure 400
@@ -40,7 +41,7 @@ func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *h
 	edgeJobFolder := handler.FileService.GetEdgeJobFolder(strconv.Itoa(edgeJobID))
 	err = handler.FileService.RemoveDirectory(edgeJobFolder)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove the files associated to the Edge job on the filesystem", err)
+		log.Warn().Err(err).Msg("Unable to remove the files associated to the Edge job on the filesystem")
 	}
 
 	handler.ReverseTunnelService.RemoveEdgeJob(edgeJob.ID)

api/http/handler/edgejobs/edgejob_file.go

@@ -20,7 +20,7 @@ type edgeJobFileResponse struct {
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
+// @param id path int true "EdgeJob Id"
 // @success 200 {object} edgeJobFileResponse
 // @failure 500
 // @failure 400

api/http/handler/edgejobs/edgejob_inspect.go

@@ -21,7 +21,7 @@ type edgeJobInspectResponse struct {
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
+// @param id path int true "EdgeJob Id"
 // @success 200 {object} portainer.EdgeJob
 // @failure 500
 // @failure 400

api/http/handler/edgejobs/edgejob_tasklogs_clear.go

@@ -19,8 +19,8 @@ import (
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
-// @param taskID path string true "Task Id"
+// @param id path int true "EdgeJob Id"
+// @param taskID path int true "Task Id"
 // @success 204
 // @failure 500
 // @failure 400
@@ -44,22 +44,33 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
 		return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
 	}
 
+	err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
+	if err != nil {
+		return httperror.InternalServerError("Unable to clear log file from disk", err)
+	}
+
 	endpointID := portainer.EndpointID(taskID)
 	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
 	if err != nil {
 		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
 	}
 
-	if slices.Contains(endpointsFromGroups, endpointID) {
-		edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
-			CollectLogs: false,
-			LogsStatus:  portainer.EdgeJobLogsStatusIdle,
+	err = handler.DataStore.EdgeJob().UpdateEdgeJobFunc(edgeJob.ID, func(j *portainer.EdgeJob) {
+		if slices.Contains(endpointsFromGroups, endpointID) {
+			j.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+				CollectLogs: false,
+				LogsStatus:  portainer.EdgeJobLogsStatusIdle,
+			}
+		} else {
+			meta := j.Endpoints[endpointID]
+			meta.CollectLogs = false
+			meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
+			j.Endpoints[endpointID] = meta
 		}
-	} else {
-		meta := edgeJob.Endpoints[endpointID]
-		meta.CollectLogs = false
-		meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
-		edgeJob.Endpoints[endpointID] = meta
+	})
+
+	if err != nil {
+		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
 	}
 
 	err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
@@ -74,10 +85,5 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
 
 	handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
 
-	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
-	}
-
 	return response.Empty(w)
 }

api/http/handler/edgejobs/edgejob_tasklogs_collect.go

@@ -7,6 +7,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/slices"
 )
@@ -18,8 +19,8 @@ import (
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
-// @param taskID path string true "Task Id"
+// @param id path int true "EdgeJob Id"
+// @param taskID path int true "Task Id"
 // @success 204
 // @failure 500
 // @failure 400
@@ -36,46 +37,58 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
 		return httperror.BadRequest("Invalid Task identifier route variable", err)
 	}
 
-	edgeJob, err := handler.DataStore.EdgeJob().EdgeJob(portainer.EdgeJobID(edgeJobID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
-	}
-
-	endpointID := portainer.EndpointID(taskID)
-	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
-	if err != nil {
-		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
-	}
-
-	if slices.Contains(endpointsFromGroups, endpointID) {
-		edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
-			CollectLogs: true,
-			LogsStatus:  portainer.EdgeJobLogsStatusPending,
+	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		edgeJob, err := tx.EdgeJob().EdgeJob(portainer.EdgeJobID(edgeJobID))
+		if handler.DataStore.IsErrObjectNotFound(err) {
+			return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
+		} else if err != nil {
+			return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
 		}
-	} else {
-		meta := edgeJob.Endpoints[endpointID]
-		meta.CollectLogs = true
-		meta.LogsStatus = portainer.EdgeJobLogsStatusPending
-		edgeJob.Endpoints[endpointID] = meta
-	}
 
-	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+		endpointID := portainer.EndpointID(taskID)
+		endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
+
+		if slices.Contains(endpointsFromGroups, endpointID) {
+			edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+				CollectLogs: true,
+				LogsStatus:  portainer.EdgeJobLogsStatusPending,
+			}
+		} else {
+			meta := edgeJob.Endpoints[endpointID]
+			meta.CollectLogs = true
+			meta.LogsStatus = portainer.EdgeJobLogsStatusPending
+			edgeJob.Endpoints[endpointID] = meta
+		}
+
+		err = tx.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
+		}
+
+		endpoint, err := tx.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve environment from the database", err)
+		}
+
+		if endpoint.Edge.AsyncMode {
+			return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
+		}
+
+		handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
+
+		return nil
+	})
+
 	if err != nil {
-		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
-	}
+		if httpErr, ok := err.(*httperror.HandlerError); ok {
+			return httpErr
+		}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve environment from the database", err)
+		return httperror.InternalServerError("Unexpected error", err)
 	}
 
-	if endpoint.Edge.AsyncMode {
-		return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
-	}
-
-	handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
-
 	return response.Empty(w)
 }

api/http/handler/edgejobs/edgejob_tasklogs_inspect.go

@@ -20,8 +20,8 @@ type fileResponse struct {
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
-// @param taskID path string true "Task Id"
+// @param id path int true "EdgeJob Id"
+// @param taskID path int true "Task Id"
 // @success 200 {object} fileResponse
 // @failure 500
 // @failure 400

api/http/handler/edgejobs/edgejob_tasks_list.go

@@ -25,7 +25,7 @@ type taskContainer struct {
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeJob Id"
+// @param id path int true "EdgeJob Id"
 // @success 200 {array} taskContainer
 // @failure 500
 // @failure 400

api/http/handler/edgejobs/edgejob_update.go

@@ -41,7 +41,7 @@ func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
 // @security jwt
 // @accept json
 // @produce json
-// @param id path string true "EdgeJob Id"
+// @param id path int true "EdgeJob Id"
 // @param body body edgeJobUpdatePayload true "EdgeGroup data"
 // @success 200 {object} portainer.EdgeJob
 // @failure 500

api/http/handler/edgestacks/edgestack_create.go

@@ -201,6 +201,8 @@ type swarmStackFromGitRepositoryPayload struct {
 	Registries []portainer.RegistryID
 	// Uses the manifest's namespaces instead of the default one
 	UseManifestNamespaces bool
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -247,6 +249,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request, dryru
 		URL:            payload.RepositoryURL,
 		ReferenceName:  payload.RepositoryReferenceName,
 		ConfigFilePath: payload.FilePathInRepository,
+		TLSSkipVerify:  payload.TLSSkipVerify,
 	}
 
 	if payload.RepositoryAuthentication {
@@ -345,7 +348,7 @@ func (handler *Handler) storeManifestFromGitRepository(stackFolder string, relat
 		repositoryPassword = repositoryConfig.Authentication.Password
 	}
 
-	err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword)
+	err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword, repositoryConfig.TLSSkipVerify)
 	if err != nil {
 		return "", "", "", err
 	}

api/http/handler/edgestacks/edgestack_delete.go

@@ -15,7 +15,7 @@ import (
 // @tags edge_stacks
 // @security ApiKeyAuth
 // @security jwt
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @success 204
 // @failure 500
 // @failure 400

api/http/handler/edgestacks/edgestack_file.go

@@ -20,7 +20,7 @@ type stackFileResponse struct {
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @success 200 {object} stackFileResponse
 // @failure 500
 // @failure 400

api/http/handler/edgestacks/edgestack_inspect.go

@@ -16,7 +16,7 @@ import (
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
 // @failure 400

api/http/handler/edgestacks/edgestack_status_delete.go

@@ -15,7 +15,7 @@ import (
 // @description Authorized only if the request is done by an Edge Environment(Endpoint)
 // @tags edge_stacks
 // @produce json
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
 // @failure 400

api/http/handler/edgestacks/edgestack_status_update.go

@@ -40,7 +40,7 @@ func (payload *updateStatusPayload) Validate(r *http.Request) error {
 // @tags edge_stacks
 // @accept json
 // @produce json
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
 // @failure 400

api/http/handler/edgestacks/edgestack_test.go

@@ -18,32 +18,12 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/edge/edgestacks"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
 
 	"github.com/pkg/errors"
 )
 
-type gitService struct {
-	cloneErr error
-	id       string
-}
-
-func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
-	return g.cloneErr
-}
-
-func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	return g.id, nil
-}
-
-func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
-	return nil, nil
-}
-
-func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
-	return nil, nil
-}
-
 // Helpers
 func setupHandler(t *testing.T) (*Handler, string, func()) {
 	t.Helper()
@@ -98,7 +78,7 @@ func setupHandler(t *testing.T) (*Handler, string, func()) {
 		t.Fatal(err)
 	}
 
-	handler.GitService = &gitService{errors.New("Clone error"), "git-service-id"}
+	handler.GitService = testhelpers.NewGitService(errors.New("Clone error"), "git-service-id")
 
 	return handler, rawAPIKey, storeTeardown
 }

api/http/handler/edgestacks/edgestack_update.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/rs/zerolog/log"
 )
 
 type updateEdgeStackPayload struct {
@@ -41,7 +42,7 @@ func (payload *updateEdgeStackPayload) Validate(r *http.Request) error {
 // @security jwt
 // @accept json
 // @produce json
-// @param id path string true "EdgeStack Id"
+// @param id path int true "EdgeStack Id"
 // @param body body updateEdgeStackPayload true "EdgeStack data"
 // @success 200 {object} portainer.EdgeStack
 // @failure 500
@@ -135,7 +136,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		// deployment type was changed - need to delete the old file
 		err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 		if err != nil {
-			return httperror.InternalServerError("Unable to clear old files", err)
+			log.Warn().Err(err).Msg("Unable to clear old files")
 		}
 
 		stack.EntryPoint = ""

api/http/handler/endpointedge/endpoint_edgejob_logs.go

@@ -25,8 +25,8 @@ func (payload *logsPayload) Validate(r *http.Request) error {
 // @tags edge, endpoints
 // @accept json
 // @produce json
-// @param id path string true "environment(endpoint) Id"
-// @param jobID path string true "Job Id"
+// @param id path int true "environment(endpoint) Id"
+// @param jobID path int true "Job Id"
 // @success 200
 // @failure 500
 // @failure 400

api/http/handler/endpointedge/endpoint_edgestack_inspect.go

@@ -25,8 +25,8 @@ type configResponse struct {
 // @tags edge, endpoints, edge_stacks
 // @accept json
 // @produce json
-// @param id path string true "environment(endpoint) Id"
-// @param stackId path string true "EdgeStack Id"
+// @param id path int true "environment(endpoint) Id"
+// @param stackId path int true "EdgeStack Id"
 // @success 200 {object} configResponse
 // @failure 500
 // @failure 400

api/http/handler/endpoints/endpoint_agent_browse_docs.go

@@ -0,0 +1,25 @@
+package endpoints
+
+/// This feature is implemented in the agent API and not directly here.
+/// However, it's proxied.  So we document it here.
+
+// @summary Upload a file under a specific path on the file system of an environment (endpoint)
+// @description Use this environment(endpoint) to upload TLS files.
+// @description **Access policy**: administrator
+// @tags endpoints
+// @security ApiKeyAuth
+// @security jwt
+// @accept multipart/form-data
+// @produce json
+// @param id path int true "Environment(Endpoint) identifier"
+// @param volumeID query string false "Optional volume identifier to upload the file"
+// @param Path formData string true "The destination path to upload the file to"
+// @param file formData file true "The file to upload"
+// @success 204 "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /endpoints/{id}/docker/v2/browse/put [post]
+func _fileBrowseFileUploadV2() {
+	// dummy function to make swag pick up the above docs for the following REST call
+	// POST request on /browse/put?volumeID=:id
+}

api/http/handler/endpoints/endpoint_create.go

@@ -164,9 +164,6 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 	checkinInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "CheckinInterval", true)
 	payload.EdgeCheckinInterval = checkinInterval
 
-	isEdgeDevice, _ := request.RetrieveBooleanMultiPartFormValue(r, "IsEdgeDevice", true)
-	payload.IsEdgeDevice = isEdgeDevice
-
 	return nil
 }
 
@@ -180,23 +177,22 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @accept multipart/form-data
 // @produce json
 // @param Name formData string true "Name that will be used to identify this environment(endpoint) (example: my-environment)"
-// @param EndpointCreationType formData integer true "Environment(Endpoint) type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5)
-// @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine)". Cannot be empty if EndpointCreationType is set to 4 (Edge agent environment)
+// @param EndpointCreationType formData integer true "Environment(Endpoint) type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment)" Enum(1,2,3,4,5)
+// @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine). Cannot be empty if EndpointCreationType is set to 4 (Edge agent environment)"
 // @param PublicURL formData string false "URL or IP address where exposed containers will be reachable. Defaults to URL if not specified (example: docker.mydomain.tld:2375)"
 // @param GroupID formData int false "Environment(Endpoint) group identifier. If not specified will default to 1 (unassigned)."
-// @param TLS formData bool false "Require TLS to connect against this environment(endpoint)"
-// @param TLSSkipVerify formData bool false "Skip server verification when using TLS"
-// @param TLSSkipClientVerify formData bool false "Skip client verification when using TLS"
+// @param TLS formData bool false "Require TLS to connect against this environment(endpoint). Must be true if EndpointCreationType is set to 2 (Agent environment)"
+// @param TLSSkipVerify formData bool false "Skip server verification when using TLS. Must be true if EndpointCreationType is set to 2 (Agent environment)"
+// @param TLSSkipClientVerify formData bool false "Skip client verification when using TLS. Must be true if EndpointCreationType is set to 2 (Agent environment)"
 // @param TLSCACertFile formData file false "TLS CA certificate file"
 // @param TLSCertFile formData file false "TLS client certificate file"
 // @param TLSKeyFile formData file false "TLS client key file"
 // @param AzureApplicationID formData string false "Azure application ID. Required if environment(endpoint) type is set to 3"
 // @param AzureTenantID formData string false "Azure tenant ID. Required if environment(endpoint) type is set to 3"
 // @param AzureAuthenticationKey formData string false "Azure authentication key. Required if environment(endpoint) type is set to 3"
-// @param TagIDs formData []int false "List of tag identifiers to which this environment(endpoint) is associated"
+// @param TagIds formData []int false "List of tag identifiers to which this environment(endpoint) is associated"
 // @param EdgeCheckinInterval formData int false "The check in interval for edge agent (in seconds)"
 // @param EdgeTunnelServerAddress formData string true "URL or IP address that will be used to establish a reverse tunnel"
-// @param IsEdgeDevice formData bool false "Is Edge Device"
 // @param Gpus formData array false "List of GPUs"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"

api/http/handler/endpoints/endpoint_inspect.go

@@ -42,7 +42,13 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		return httperror.Forbidden("Permission denied to access environment", err)
 	}
 
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve settings from the database", err)
+	}
+
 	hideFields(endpoint)
+	endpointutils.UpdateEdgeEndpointHeartbeat(endpoint, settings)
 	endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 
 	if !excludeSnapshot(r) {

api/http/handler/endpoints/endpoint_list.go

@@ -9,6 +9,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
@@ -42,8 +43,8 @@ const (
 // @param endpointIds query []int false "will return only these environments(endpoints)"
 // @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned"
 // @param agentVersions query []string false "will return only environments with on of these agent versions"
-// @param edgeDevice query bool false "if exists true show only edge devices, false show only regular edge endpoints. if missing, will show both types (relevant only for edge endpoints)"
-// @param edgeDeviceUntrusted query bool false "if true, show only untrusted endpoints, if false show only trusted (relevant only for edge devices, and if edgeDevice is true)"
+// @param edgeAsync query bool false "if exists true show only edge async agents, false show only standard edge agents. if missing, will show both types (relevant only for edge agents)"
+// @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
 // @param name query string false "will return only environments(endpoints) with this name"
 // @success 200 {array} portainer.Endpoint "Endpoints"
 // @failure 500 "Server error"
@@ -103,6 +104,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 			paginatedEndpoints[idx].EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
 		}
 		paginatedEndpoints[idx].QueryDate = time.Now().Unix()
+		endpointutils.UpdateEdgeEndpointHeartbeat(&paginatedEndpoints[idx], settings)
 		if !query.excludeSnapshots {
 			err = handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx])
 			if err != nil {

api/http/handler/endpoints/endpoint_list_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/testhelpers"
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -104,59 +103,58 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
 	}
 }
 
-func Test_endpointList_edgeDeviceFilter(t *testing.T) {
+func Test_endpointList_edgeFilter(t *testing.T) {
 
-	trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularEndpoint := portainer.Endpoint{ID: 5, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.DockerEnvironment}
+	trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
 
 	handler, teardown := setup(t, []portainer.Endpoint{
-		trustedEdgeDevice,
-		untrustedEdgeDevice,
-		regularUntrustedEdgeEndpoint,
-		regularTrustedEdgeEndpoint,
+		trustedEdgeAsync,
+		untrustedEdgeAsync,
+		regularUntrustedEdgeStandard,
+		regularTrustedEdgeStandard,
 		regularEndpoint,
 	})
 
 	defer teardown()
 
-	type endpointListEdgeDeviceTest struct {
+	type endpointListEdgeTest struct {
 		endpointListTest
-		edgeDevice          *bool
+		edgeAsync           *bool
 		edgeDeviceUntrusted bool
 	}
 
-	tests := []endpointListEdgeDeviceTest{
+	tests := []endpointListEdgeTest{
 		{
 			endpointListTest: endpointListTest{
-				"should show all endpoints except of the untrusted devices",
-				[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID, regularEndpoint.ID},
+				"should show all endpoints expect of the untrusted devices",
+				[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID, regularEndpoint.ID},
 			},
-			edgeDevice: nil,
 		},
 		{
 			endpointListTest: endpointListTest{
-				"should show only trusted edge devices and regular endpoints",
-				[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
+				"should show only trusted edge async agents and regular endpoints",
+				[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
 			},
-			edgeDevice: BoolAddr(true),
+			edgeAsync: BoolAddr(true),
 		},
 		{
 			endpointListTest: endpointListTest{
 				"should show only untrusted edge devices and regular endpoints",
-				[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
+				[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
 			},
-			edgeDevice:          BoolAddr(true),
+			edgeAsync:           BoolAddr(true),
 			edgeDeviceUntrusted: true,
 		},
 		{
 			endpointListTest: endpointListTest{
 				"should show no edge devices",
-				[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+				[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
 			},
-			edgeDevice: BoolAddr(false),
+			edgeAsync: BoolAddr(false),
 		},
 	}
 
@@ -165,8 +163,8 @@ func Test_endpointList_edgeDeviceFilter(t *testing.T) {
 			is := assert.New(t)
 
 			query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted)
-			if test.edgeDevice != nil {
-				query += fmt.Sprintf("edgeDevice=%v&", *test.edgeDevice)
+			if test.edgeAsync != nil {
+				query += fmt.Sprintf("edgeAsync=%v&", *test.edgeAsync)
 			}
 
 			req := buildEndpointListRequest(query)
@@ -198,7 +196,7 @@ func setup(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, tear
 	err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
-	bouncer := helper.NewTestRequestBouncer()
+	bouncer := testhelpers.NewTestRequestBouncer()
 	handler = NewHandler(bouncer, nil)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()

api/http/handler/endpoints/endpoint_settings_update.go

@@ -28,6 +28,10 @@ type endpointSettingsUpdatePayload struct {
 	AllowSysctlSettingForRegularUsers *bool `json:"allowSysctlSettingForRegularUsers" example:"true"`
 	// Whether host management features are enabled
 	EnableHostManagementFeatures *bool `json:"enableHostManagementFeatures" example:"true"`
+
+	EnableGPUManagement *bool `json:"enableGPUManagement" example:"false"`
+
+	Gpus []portainer.Pair `json:"gpus"`
 }
 
 func (payload *endpointSettingsUpdatePayload) Validate(r *http.Request) error {
@@ -107,6 +111,14 @@ func (handler *Handler) endpointSettingsUpdate(w http.ResponseWriter, r *http.Re
 		securitySettings.EnableHostManagementFeatures = *payload.EnableHostManagementFeatures
 	}
 
+	if payload.EnableGPUManagement != nil {
+		endpoint.EnableGPUManagement = *payload.EnableGPUManagement
+	}
+
+	if payload.Gpus != nil {
+		endpoint.Gpus = payload.Gpus
+	}
+
 	endpoint.SecuritySettings = securitySettings
 
 	err = handler.DataStore.Endpoint().UpdateEndpoint(portainer.EndpointID(endpointID), endpoint)

api/http/handler/endpoints/filter.go

@@ -15,14 +15,15 @@ import (
 )
 
 type EnvironmentsQuery struct {
-	search              string
-	types               []portainer.EndpointType
-	tagIds              []portainer.TagID
-	endpointIds         []portainer.EndpointID
-	tagsPartialMatch    bool
-	groupIds            []portainer.EndpointGroupID
-	status              []portainer.EndpointStatus
-	edgeDevice          *bool
+	search           string
+	types            []portainer.EndpointType
+	tagIds           []portainer.TagID
+	endpointIds      []portainer.EndpointID
+	tagsPartialMatch bool
+	groupIds         []portainer.EndpointGroupID
+	status           []portainer.EndpointStatus
+	// if edgeAsync not nil, will filter edge endpoints based on this value
+	edgeAsync           *bool
 	edgeDeviceUntrusted bool
 	excludeSnapshots    bool
 	name                string
@@ -66,11 +67,10 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 
 	name, _ := request.RetrieveQueryParameter(r, "name", true)
 
-	edgeDeviceParam, _ := request.RetrieveQueryParameter(r, "edgeDevice", true)
-
-	var edgeDevice *bool
-	if edgeDeviceParam != "" {
-		edgeDevice = BoolAddr(edgeDeviceParam == "true")
+	var edgeAsync *bool
+	edgeAsyncParam, _ := request.RetrieveQueryParameter(r, "edgeAsync", true)
+	if edgeAsyncParam != "" {
+		edgeAsync = BoolAddr(edgeAsyncParam == "true")
 	}
 
 	edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true)
@@ -85,7 +85,7 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 		tagsPartialMatch:    tagsPartialMatch,
 		groupIds:            groupIDs,
 		status:              status,
-		edgeDevice:          edgeDevice,
+		edgeAsync:           edgeAsync,
 		edgeDeviceUntrusted: edgeDeviceUntrusted,
 		excludeSnapshots:    excludeSnapshots,
 		name:                name,
@@ -108,15 +108,26 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End
 		filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
 	}
 
-	if query.edgeDevice != nil {
-		filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, *query.edgeDevice, query.edgeDeviceUntrusted)
-	} else {
-		// If the edgeDevice parameter is not set, we need to filter out the untrusted edge devices
+	// filter async edge environments
+	if query.edgeAsync != nil {
 		filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
-			return !endpoint.IsEdgeDevice || endpoint.UserTrusted
+			if !endpointutils.IsEdgeEndpoint(&endpoint) {
+				return true
+			}
+
+			return endpoint.Edge.AsyncMode == *query.edgeAsync
 		})
 	}
 
+	// filter edge environments by trusted/untrusted
+	filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
+		if !endpointutils.IsEdgeEndpoint(&endpoint) {
+			return true
+		}
+
+		return endpoint.UserTrusted == !query.edgeDeviceUntrusted
+	})
+
 	if len(query.status) > 0 {
 		filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings)
 	}
@@ -274,30 +285,6 @@ func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []port
 	return endpoints[:n]
 }
 
-func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDevice bool, untrusted bool) []portainer.Endpoint {
-	n := 0
-	for _, endpoint := range endpoints {
-		if shouldReturnEdgeDevice(endpoint, edgeDevice, untrusted) {
-			endpoints[n] = endpoint
-			n++
-		}
-	}
-
-	return endpoints[:n]
-}
-
-func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceParam bool, untrustedParam bool) bool {
-	if !endpointutils.IsEdgeEndpoint(&endpoint) {
-		return true
-	}
-
-	if !edgeDeviceParam {
-		return !endpoint.IsEdgeDevice
-	}
-
-	return endpoint.IsEdgeDevice && endpoint.UserTrusted == !untrustedParam
-}
-
 func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
 	tags := make([]string, 0, len(tagIDs))
 

api/http/handler/endpoints/filter_test.go

@@ -6,7 +6,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/internal/testhelpers"
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -74,19 +73,19 @@ func Test_Filter_AgentVersion(t *testing.T) {
 	runTests(tests, t, handler, endpoints)
 }
 
-func Test_Filter_edgeDeviceFilter(t *testing.T) {
+func Test_Filter_edgeFilter(t *testing.T) {
 
-	trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
 	regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
 
 	endpoints := []portainer.Endpoint{
-		trustedEdgeDevice,
-		untrustedEdgeDevice,
-		regularUntrustedEdgeEndpoint,
-		regularTrustedEdgeEndpoint,
+		trustedEdgeAsync,
+		untrustedEdgeAsync,
+		regularUntrustedEdgeStandard,
+		regularTrustedEdgeStandard,
 		regularEndpoint,
 	}
 
@@ -96,32 +95,32 @@ func Test_Filter_edgeDeviceFilter(t *testing.T) {
 
 	tests := []filterTest{
 		{
-			"should show all edge endpoints except of the untrusted devices",
-			[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+			"should show all edge endpoints except of the untrusted edge",
+			[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID},
 			EnvironmentsQuery{
 				types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment},
 			},
 		},
 		{
 			"should show only trusted edge devices and other regular endpoints",
-			[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
+			[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
 			EnvironmentsQuery{
-				edgeDevice: BoolAddr(true),
+				edgeAsync: BoolAddr(true),
 			},
 		},
 		{
 			"should show only untrusted edge devices and other regular endpoints",
-			[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
+			[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
 			EnvironmentsQuery{
-				edgeDevice:          BoolAddr(true),
+				edgeAsync:           BoolAddr(true),
 				edgeDeviceUntrusted: true,
 			},
 		},
 		{
 			"should show no edge devices",
-			[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+			[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
 			EnvironmentsQuery{
-				edgeDevice: BoolAddr(false),
+				edgeAsync: BoolAddr(false),
 			},
 		},
 	}
@@ -168,7 +167,7 @@ func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) (handler *Han
 	err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
-	bouncer := helper.NewTestRequestBouncer()
+	bouncer := testhelpers.NewTestRequestBouncer()
 	handler = NewHandler(bouncer, nil)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()

api/http/handler/handler.go

@@ -82,7 +82,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.18.0
+// @version 2.18.2
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -97,7 +97,7 @@ type Handler struct {
 
 // @securitydefinitions.apikey ApiKeyAuth
 // @in header
-// @name Authorization
+// @name x-api-key
 
 // @securitydefinitions.apikey jwt
 // @in header
@@ -107,6 +107,8 @@ type Handler struct {
 // @tag.description Authenticate against Portainer HTTP API
 // @tag.name custom_templates
 // @tag.description Manage Custom Templates
+// @tag.name edge
+// @tag.description Manage Edge related environment(endpoint) settings
 // @tag.name edge_groups
 // @tag.description Manage Edge Groups
 // @tag.name edge_jobs
@@ -115,8 +117,6 @@ type Handler struct {
 // @tag.description Manage Edge Stacks
 // @tag.name edge_templates
 // @tag.description Manage Edge Templates
-// @tag.name edge
-// @tag.description Manage Edge related environment(endpoint) settings
 // @tag.name endpoints
 // @tag.description Manage Docker environments(endpoints)
 // @tag.name endpoint_groups
@@ -133,8 +133,14 @@ type Handler struct {
 // @tag.description Manage roles
 // @tag.name settings
 // @tag.description Manage Portainer settings
-// @tag.name users
-// @tag.description Manage users
+// @tag.name ssl
+// @tag.description Manage ssl settings
+// @tag.name stacks
+// @tag.description Manage stacks
+// @tag.name status
+// @tag.description Information about the Portainer instance
+// @tag.name system
+// @tag.description Manage Portainer system
 // @tag.name tags
 // @tag.description Manage tags
 // @tag.name teams
@@ -143,20 +149,14 @@ type Handler struct {
 // @tag.description Manage team memberships
 // @tag.name templates
 // @tag.description Manage App Templates
-// @tag.name stacks
-// @tag.description Manage stacks
-// @tag.name ssl
-// @tag.description Manage ssl settings
+// @tag.name users
+// @tag.description Manage users
 // @tag.name upload
 // @tag.description Upload files
 // @tag.name webhooks
 // @tag.description Manage webhooks
 // @tag.name websocket
 // @tag.description Create exec sessions using websockets
-// @tag.name status
-// @tag.description Information about the Portainer instance
-// @tag.name system
-// @tag.description Manage Portainer system
 
 // ServeHTTP delegates a request to the appropriate subhandler.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

api/http/handler/helm/helm_delete.go

@@ -18,7 +18,7 @@ import (
 // @security jwt
 // @param id path int true "Environment(Endpoint) identifier"
 // @param release path string true "The name of the release/application to uninstall"
-// @param namespace query string true "An optional namespace"
+// @param namespace query string false "An optional namespace"
 // @success 204 "Success"
 // @failure 400 "Invalid environment(endpoint) id or bad request"
 // @failure 401 "Unauthorized"

api/http/handler/helm/helm_list.go

@@ -20,9 +20,9 @@ import (
 // @accept json
 // @produce json
 // @param id path int true "Environment(Endpoint) identifier"
-// @param namespace query string true "specify an optional namespace"
-// @param filter query string true "specify an optional filter"
-// @param selector query string true "specify an optional selector"
+// @param namespace query string false "specify an optional namespace"
+// @param filter query string false "specify an optional filter"
+// @param selector query string false "specify an optional selector"
 // @success 200 {array} release.ReleaseElement "Success"
 // @failure 400 "Invalid environment(endpoint) identifier"
 // @failure 401 "Unauthorized"

api/http/handler/helm/user_helm_repos.go

@@ -25,7 +25,7 @@ type addHelmRepoUrlPayload struct {
 }
 
 func (p *addHelmRepoUrlPayload) Validate(_ *http.Request) error {
-	return libhelm.ValidateHelmRepositoryURL(p.URL)
+	return libhelm.ValidateHelmRepositoryURL(p.URL, nil)
 }
 
 // @id HelmUserRepositoryCreate

api/http/handler/kubernetes/services.go

@@ -37,7 +37,15 @@ func (handler *Handler) getKubernetesServices(w http.ResponseWriter, r *http.Req
 		)
 	}
 
-	services, err := cli.GetServices(namespace)
+	lookup, err := request.RetrieveBooleanQueryParameter(r, "lookupapplications", true)
+	if err != nil {
+		return httperror.BadRequest(
+			"Invalid lookupapplications query parameter",
+			err,
+		)
+	}
+
+	services, err := cli.GetServices(namespace, lookup)
 	if err != nil {
 		return httperror.InternalServerError(
 			"Unable to retrieve services",

api/http/handler/settings/settings_public.go

@@ -40,8 +40,6 @@ type publicSettingsResponse struct {
 	IsAMTEnabled bool
 
 	Edge struct {
-		// Whether the device has been started in edge async mode
-		AsyncMode bool
 		// The ping interval for edge agent - used in edge async mode [seconds]
 		PingInterval int `json:"PingInterval" example:"60"`
 		// The snapshot interval for edge agent - used in edge async mode [seconds]
@@ -86,7 +84,6 @@ func generatePublicSettings(appSettings *portainer.Settings) *publicSettingsResp
 		IsAMTEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.OpenAMTConfiguration.Enabled,
 	}
 
-	publicSettings.Edge.AsyncMode = appSettings.Edge.AsyncMode
 	publicSettings.Edge.PingInterval = appSettings.Edge.PingInterval
 	publicSettings.Edge.SnapshotInterval = appSettings.Edge.SnapshotInterval
 	publicSettings.Edge.CommandInterval = appSettings.Edge.CommandInterval

api/http/handler/settings/settings_update.go

@@ -143,7 +143,7 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 			newHelmRepo := strings.TrimSuffix(strings.ToLower(*payload.HelmRepositoryURL), "/")
 
 			if newHelmRepo != settings.HelmRepositoryURL && newHelmRepo != portainer.DefaultHelmRepositoryURL {
-				err := libhelm.ValidateHelmRepositoryURL(*payload.HelmRepositoryURL)
+				err := libhelm.ValidateHelmRepositoryURL(*payload.HelmRepositoryURL, nil)
 				if err != nil {
 					return httperror.BadRequest("Invalid Helm repository URL. Must correspond to a valid URL format", err)
 				}

api/http/handler/stacks/create_compose_stack.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
@@ -156,14 +157,16 @@ type composeStackFromGitRepositoryPayload struct {
 	// Applicable when deploying with multiple stack files
 	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
 	// Optional auto update configuration
-	AutoUpdate *portainer.StackAutoUpdate
+	AutoUpdate *portainer.AutoUpdateSettings
 	// A list of environment(endpoint) variables used during stack deployment
 	Env []portainer.Pair
 	// Whether the stack is from a app template
 	FromAppTemplate bool `example:"false"`
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
-func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
+func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool, repoSkipSSLVerify bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		Name: name,
 		RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{
@@ -172,6 +175,7 @@ func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoU
 			Authentication: repoAuthentication,
 			Username:       repoUsername,
 			Password:       repoPassword,
+			TLSSkipVerify:  repoSkipSSLVerify,
 		},
 		ComposeFile:     composeFile,
 		AdditionalFiles: additionalFiles,
@@ -191,7 +195,7 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil
@@ -257,7 +261,9 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		payload.AdditionalFiles,
 		payload.AutoUpdate,
 		payload.Env,
-		payload.FromAppTemplate)
+		payload.FromAppTemplate,
+		payload.TLSSkipVerify,
+	)
 
 	composeStackBuilder := stackbuilders.CreateComposeStackGitBuilder(securityContext,
 		handler.DataStore,

api/http/handler/stacks/create_kubernetes_stack.go

@@ -11,6 +11,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/git/update"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
@@ -44,10 +45,12 @@ type kubernetesGitDeploymentPayload struct {
 	RepositoryPassword       string
 	ManifestFile             string
 	AdditionalFiles          []string
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
-func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate) stackbuilders.StackPayload {
+func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, repoSkipSSLVerify bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		StackName: name,
 		RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{
@@ -56,6 +59,7 @@ func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsern
 			Authentication: repoAuthentication,
 			Username:       repoUsername,
 			Password:       repoPassword,
+			TLSSkipVerify:  repoSkipSSLVerify,
 		},
 		Namespace:       namespace,
 		ComposeFormat:   composeFormat,
@@ -101,7 +105,7 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.ManifestFile) {
 		return errors.New("Invalid manifest file in repository")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	if govalidator.IsNull(payload.StackName) {
@@ -202,7 +206,9 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 		payload.Namespace,
 		payload.ManifestFile,
 		payload.AdditionalFiles,
-		payload.AutoUpdate)
+		payload.AutoUpdate,
+		payload.TLSSkipVerify,
+	)
 
 	k8sStackBuilder := stackbuilders.CreateKubernetesStackGitBuilder(handler.DataStore,
 		handler.FileService,

api/http/handler/stacks/create_swarm_stack.go

@@ -10,6 +10,7 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
 	"github.com/portainer/portainer/api/stacks/stackutils"
@@ -115,7 +116,9 @@ type swarmStackFromGitRepositoryPayload struct {
 	// Applicable when deploying with multiple stack files
 	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
 	// Optional auto update configuration
-	AutoUpdate *portainer.StackAutoUpdate
+	AutoUpdate *portainer.AutoUpdateSettings
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -131,13 +134,13 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil
 }
 
-func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
+func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool, repoSkipSSLVerify bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		Name:    name,
 		SwarmID: swarmID,
@@ -147,6 +150,7 @@ func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference
 			Authentication: repoAuthentication,
 			Username:       repoUsername,
 			Password:       repoPassword,
+			TLSSkipVerify:  repoSkipSSLVerify,
 		},
 		ComposeFile:     composeFile,
 		AdditionalFiles: additionalFiles,
@@ -200,7 +204,9 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 		payload.AdditionalFiles,
 		payload.AutoUpdate,
 		payload.Env,
-		payload.FromAppTemplate)
+		payload.FromAppTemplate,
+		payload.TLSSkipVerify,
+	)
 
 	swarmStackBuilder := stackbuilders.CreateSwarmStackGitBuilder(securityContext,
 		handler.DataStore,

api/http/handler/stacks/stack_delete.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackutils"
+	"github.com/rs/zerolog/log"
 )
 
 // @id StackDelete
@@ -136,7 +137,7 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 
 	err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove stack files from disk", err)
+		log.Warn().Err(err).Msg("Unable to remove stack files from disk")
 	}
 
 	return response.Empty(w)

api/http/handler/stacks/stack_update_git.go

@@ -10,6 +10,7 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/git/update"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
@@ -17,17 +18,18 @@ import (
 )
 
 type stackGitUpdatePayload struct {
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
 	Env                      []portainer.Pair
 	Prune                    bool
 	RepositoryReferenceName  string
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
+	TLSSkipVerify            bool
 }
 
 func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil
@@ -137,6 +139,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 
 	//update retrieved stack data based on the payload
 	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+	stack.GitConfig.TLSSkipVerify = payload.TLSSkipVerify
 	stack.AutoUpdate = payload.AutoUpdate
 	stack.Env = payload.Env
 	stack.UpdatedBy = user.Username
@@ -150,6 +153,9 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 
 	if payload.RepositoryAuthentication {
 		password := payload.RepositoryPassword
+
+		// When the existing stack is using the custom username/password and the password is not updated,
+		// the stack should keep using the saved username/password
 		if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
 			password = stack.GitConfig.Authentication.Password
 		}
@@ -157,7 +163,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 			Username: payload.RepositoryUsername,
 			Password: password,
 		}
-		_, err = handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password)
+		_, err = handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password, stack.GitConfig.TLSSkipVerify)
 		if err != nil {
 			return httperror.InternalServerError("Unable to fetch git repository", err)
 		}

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -1,7 +1,6 @@
 package stacks
 
 import (
-	"fmt"
 	"net/http"
 	"time"
 
@@ -10,14 +9,12 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackutils"
-
-	"github.com/rs/zerolog/log"
 )
 
 type stackGitRedployPayload struct {
@@ -138,45 +135,41 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
-	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
-	if err != nil {
-		return httperror.InternalServerError("Unable to move git repository directory", err)
-	}
-
 	repositoryUsername := ""
 	repositoryPassword := ""
 	if payload.RepositoryAuthentication {
 		repositoryPassword = payload.RepositoryPassword
+
+		// When the existing stack is using the custom username/password and the password is not updated,
+		// the stack should keep using the saved username/password
 		if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
 			repositoryPassword = stack.GitConfig.Authentication.Password
 		}
 		repositoryUsername = payload.RepositoryUsername
 	}
 
-	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
-		if restoreError != nil {
-			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
-		}
-
-		return httperror.InternalServerError("Unable to clone git repository", err)
+	cloneOptions := git.CloneOptions{
+		ProjectPath:   stack.ProjectPath,
+		URL:           stack.GitConfig.URL,
+		ReferenceName: stack.GitConfig.ReferenceName,
+		Username:      repositoryUsername,
+		Password:      repositoryPassword,
+		TLSSkipVerify: stack.GitConfig.TLSSkipVerify,
 	}
 
-	defer func() {
-		err = handler.FileService.RemoveDirectory(backupProjectPath)
-		if err != nil {
-			log.Warn().Err(err).Msg("unable to remove git repository directory")
-		}
-	}()
+	clean, err := git.CloneWithBackup(handler.GitService, handler.FileService, cloneOptions)
+	if err != nil {
+		return httperror.InternalServerError("Unable to clone git repository directory", err)
+	}
+
+	defer clean()
 
 	httpErr := handler.deployStack(r, stack, payload.PullImage, endpoint)
 	if httpErr != nil {
 		return httpErr
 	}
 
-	newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword)
+	newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword, stack.GitConfig.TLSSkipVerify)
 	if err != nil {
 		return httperror.InternalServerError("Unable get latest commit id", errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID))
 	}

api/http/handler/stacks/update_kubernetes_stack.go

@@ -11,10 +11,10 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
 	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
-	"github.com/portainer/portainer/api/stacks/stackutils"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
@@ -30,7 +30,8 @@ type kubernetesGitStackUpdatePayload struct {
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
+	TLSSkipVerify            bool
 }
 
 func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error {
@@ -41,7 +42,7 @@ func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error
 }
 
 func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error {
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil
@@ -62,6 +63,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 		}
 
 		stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+		stack.GitConfig.TLSSkipVerify = payload.TLSSkipVerify
 		stack.AutoUpdate = payload.AutoUpdate
 
 		if payload.RepositoryAuthentication {
@@ -73,7 +75,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 				Username: payload.RepositoryUsername,
 				Password: password,
 			}
-			_, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password)
+			_, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password, stack.GitConfig.TLSSkipVerify)
 			if err != nil {
 				return httperror.InternalServerError("Unable to fetch git repository", err)
 			}

api/http/handler/stacks/webhook_invoke.go

@@ -9,7 +9,6 @@ import (
 	"github.com/portainer/portainer/api/stacks/deployments"
 
 	"github.com/gofrs/uuid"
-	"github.com/rs/zerolog/log"
 )
 
 // @id WebhookInvoke
@@ -43,8 +42,6 @@ func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *h
 			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: "Autoupdate for the stack isn't available", Err: err}
 		}
 
-		log.Error().Err(err).Msg("failed to update the stack")
-
 		return httperror.InternalServerError("Failed to update the stack", err)
 	}
 

api/http/handler/stacks/webhook_invoke_test.go

@@ -18,7 +18,8 @@ func TestHandler_webhookInvoke(t *testing.T) {
 
 	webhookID := newGuidString(t)
 	store.StackService.Create(&portainer.Stack{
-		AutoUpdate: &portainer.StackAutoUpdate{
+		ID: 1,
+		AutoUpdate: &portainer.AutoUpdateSettings{
 			Webhook: webhookID,
 		},
 	})

api/http/handler/teams/team_delete.go

@@ -17,7 +17,7 @@ import (
 // @tags teams
 // @security ApiKeyAuth
 // @security jwt
-// @param id path string true "Team Id"
+// @param id path int true "Team Id"
 // @success 204 "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"

api/http/handler/teams/team_memberships.go

@@ -19,7 +19,7 @@ import (
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param id path string true "Team Id"
+// @param id path int true "Team Id"
 // @success 200 {array} portainer.TeamMembership "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"

api/http/handler/templates/template_file.go

@@ -98,7 +98,7 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht
 
 	defer handler.cleanUp(projectPath)
 
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "")
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "", false)
 	if err != nil {
 		return httperror.InternalServerError("Unable to clone git repository", err)
 	}

api/http/handler/users/user_update.go

@@ -18,8 +18,6 @@ import (
 type themePayload struct {
 	// Color represents the color theme of the UI
 	Color *string `json:"color" example:"dark" enums:"dark,light,highcontrast,auto"`
-	// SubtleUpgradeButton indicates if the upgrade banner should be displayed in a subtle way
-	SubtleUpgradeButton *bool `json:"subtleUpgradeButton" example:"false"`
 }
 
 type userUpdatePayload struct {
@@ -33,11 +31,11 @@ type userUpdatePayload struct {
 
 func (payload *userUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.Contains(payload.Username, " ") {
-		return errors.New("Invalid username. Must not contain any whitespace")
+		return errors.New("invalid username. Must not contain any whitespace")
 	}
 
 	if payload.Role != 0 && payload.Role != 1 && payload.Role != 2 {
-		return errors.New("Invalid role value. Value must be one of: 1 (administrator) or 2 (regular user)")
+		return errors.New("invalid role value. Value must be one of: 1 (administrator) or 2 (regular user)")
 	}
 	return nil
 }
@@ -120,10 +118,6 @@ func (handler *Handler) userUpdate(w http.ResponseWriter, r *http.Request) *http
 		if payload.Theme.Color != nil {
 			user.ThemeSettings.Color = *payload.Theme.Color
 		}
-
-		if payload.Theme.SubtleUpgradeButton != nil {
-			user.ThemeSettings.SubtleUpgradeButton = *payload.Theme.SubtleUpgradeButton
-		}
 	}
 
 	if payload.Role != 0 {

api/http/models/kubernetes/application.go

@@ -0,0 +1,11 @@
+package kubernetes
+
+type (
+	K8sApplication struct {
+		UID       string            `json:",omitempty"`
+		Name      string            `json:""`
+		Namespace string            `json:",omitempty"`
+		Kind      string            `json:",omitempty"`
+		Labels    map[string]string `json:",omitempty"`
+	}
+)

api/http/models/kubernetes/ingress.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"errors"
 	"net/http"
+	"time"
 )
 
 type (
@@ -18,15 +19,17 @@ type (
 	K8sIngressControllers []K8sIngressController
 
 	K8sIngressInfo struct {
-		Name        string            `json:"Name"`
-		UID         string            `json:"UID"`
-		Type        string            `json:"Type"`
-		Namespace   string            `json:"Namespace"`
-		ClassName   string            `json:"ClassName"`
-		Annotations map[string]string `json:"Annotations"`
-		Hosts       []string          `json:"Hosts"`
-		Paths       []K8sIngressPath  `json:"Paths"`
-		TLS         []K8sIngressTLS   `json:"TLS"`
+		Name         string            `json:"Name"`
+		UID          string            `json:"UID"`
+		Type         string            `json:"Type"`
+		Namespace    string            `json:"Namespace"`
+		ClassName    string            `json:"ClassName"`
+		Annotations  map[string]string `json:"Annotations"`
+		Hosts        []string          `json:"Hosts"`
+		Paths        []K8sIngressPath  `json:"Paths"`
+		TLS          []K8sIngressTLS   `json:"TLS"`
+		Labels       map[string]string `json:"Labels,omitempty"`
+		CreationDate time.Time         `json:"CreationDate"`
 	}
 
 	K8sIngressTLS struct {

api/http/models/kubernetes/services.go

@@ -7,17 +7,23 @@ import (
 
 type (
 	K8sServiceInfo struct {
-		Name                          string              `json:"Name"`
-		UID                           string              `json:"UID"`
-		Type                          string              `json:"Type"`
-		Namespace                     string              `json:"Namespace"`
-		Annotations                   map[string]string   `json:"Annotations"`
-		CreationTimestamp             string              `json:"CreationTimestamp"`
-		Labels                        map[string]string   `json:"Labels"`
-		AllocateLoadBalancerNodePorts *bool               `json:"AllocateLoadBalancerNodePorts,omitempty"`
-		Ports                         []K8sServicePort    `json:"Ports"`
-		Selector                      map[string]string   `json:"Selector"`
-		IngressStatus                 []K8sServiceIngress `json:"IngressStatus"`
+		Name                          string
+		UID                           string
+		Type                          string
+		Namespace                     string
+		Annotations                   map[string]string
+		CreationTimestamp             string
+		Labels                        map[string]string
+		AllocateLoadBalancerNodePorts *bool `json:",omitempty"`
+		Ports                         []K8sServicePort
+		Selector                      map[string]string
+		IngressStatus                 []K8sServiceIngress `json:",omitempty"`
+
+		// serviceList screen
+		Applications []K8sApplication `json:",omitempty"`
+		ClusterIPs   []string         `json:",omitempty"`
+		ExternalName string           `json:",omitempty"`
+		ExternalIPs  []string         `json:",omitempty"`
 	}
 
 	K8sServicePort struct {
@@ -25,7 +31,7 @@ type (
 		NodePort   int    `json:"NodePort"`
 		Port       int    `json:"Port"`
 		Protocol   string `json:"Protocol"`
-		TargetPort int    `json:"TargetPort"`
+		TargetPort string `json:"TargetPort"`
 	}
 
 	K8sServiceIngress struct {

api/http/proxy/factory/docker/transport.go

@@ -395,7 +395,7 @@ func (transport *Transport) updateDefaultGitBranch(request *http.Request) error
 	remote := request.URL.Query().Get("remote")
 	if strings.HasSuffix(remote, ".git") {
 		repositoryURL := remote[:len(remote)-4]
-		latestCommitID, err := transport.gitService.LatestCommitID(repositoryURL, "", "", "")
+		latestCommitID, err := transport.gitService.LatestCommitID(repositoryURL, "", "", "", false)
 		if err != nil {
 			return err
 		}

api/http/proxy/factory/docker/transport_test.go

@@ -1,29 +1,16 @@
 package docker
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/stretchr/testify/assert"
 )
 
-type noopGitService struct{}
-
-func (s *noopGitService) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error {
-	return nil
-}
-func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	return "my-latest-commit-id", nil
-}
-func (g *noopGitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
-	return nil, nil
-}
-func (g *noopGitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
-	return nil, nil
-}
-
 func TestTransport_updateDefaultGitBranch(t *testing.T) {
 	type fields struct {
 		gitService portainer.GitService
@@ -33,8 +20,10 @@ func TestTransport_updateDefaultGitBranch(t *testing.T) {
 		request *http.Request
 	}
 
+	commitId := "my-latest-commit-id"
+
 	defaultFields := fields{
-		gitService: &noopGitService{},
+		gitService: testhelpers.NewGitService(nil, commitId),
 	}
 
 	tests := []struct {
@@ -51,7 +40,7 @@ func TestTransport_updateDefaultGitBranch(t *testing.T) {
 				request: httptest.NewRequest(http.MethodPost, "http://unixsocket/build?dockerfile=Dockerfile&remote=https://my-host.com/my-user/my-repo.git&t=my-image", nil),
 			},
 			wantErr:       false,
-			expectedQuery: "dockerfile=Dockerfile&remote=https%3A%2F%2Fmy-host.com%2Fmy-user%2Fmy-repo.git%23my-latest-commit-id&t=my-image",
+			expectedQuery: fmt.Sprintf("dockerfile=Dockerfile&remote=https%%3A%%2F%%2Fmy-host.com%%2Fmy-user%%2Fmy-repo.git%%23%s&t=my-image", commitId),
 		},
 		{
 			name:   "not append commit ID",

api/internal/edge/edgegroup.go

@@ -45,7 +45,7 @@ func EdgeGroupSet(edgeGroupIDs []portainer.EdgeGroupID) map[portainer.EdgeGroupI
 	return set
 }
 
-func GetEndpointsFromEdgeGroups(edgeGroupIDs []portainer.EdgeGroupID, datastore dataservices.DataStore) ([]portainer.EndpointID, error) {
+func GetEndpointsFromEdgeGroups(edgeGroupIDs []portainer.EdgeGroupID, datastore dataservices.DataStoreTx) ([]portainer.EndpointID, error) {
 	endpoints, err := datastore.Endpoint().Endpoints()
 	if err != nil {
 		return nil, err

api/internal/endpointutils/endpointutils.go

@@ -76,6 +76,16 @@ func EndpointSet(endpointIDs []portainer.EndpointID) map[portainer.EndpointID]bo
 }
 
 func InitialIngressClassDetection(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) {
+	if endpoint.Kubernetes.Flags.IsServerIngressClassDetected {
+		return
+	}
+	defer func() {
+		endpoint.Kubernetes.Flags.IsServerIngressClassDetected = true
+		endpointService.UpdateEndpoint(
+			portainer.EndpointID(endpoint.ID),
+			endpoint,
+		)
+	}()
 	cli, err := factory.GetKubeClient(endpoint)
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to create kubernetes client for ingress class detection")
@@ -107,6 +117,16 @@ func InitialIngressClassDetection(endpoint *portainer.Endpoint, endpointService
 }
 
 func InitialMetricsDetection(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) {
+	if endpoint.Kubernetes.Flags.IsServerMetricsDetected {
+		return
+	}
+	defer func() {
+		endpoint.Kubernetes.Flags.IsServerMetricsDetected = true
+		endpointService.UpdateEndpoint(
+			portainer.EndpointID(endpoint.ID),
+			endpoint,
+		)
+	}()
 	cli, err := factory.GetKubeClient(endpoint)
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to create kubernetes client for initial metrics detection")
@@ -118,11 +138,6 @@ func InitialMetricsDetection(endpoint *portainer.Endpoint, endpointService datas
 		return
 	}
 	endpoint.Kubernetes.Configuration.UseServerMetrics = true
-	endpoint.Kubernetes.Flags.IsServerMetricsDetected = true
-	err = endpointService.UpdateEndpoint(
-		portainer.EndpointID(endpoint.ID),
-		endpoint,
-	)
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to enable UseServerMetrics inside the database")
 		return
@@ -158,6 +173,16 @@ func storageDetect(endpoint *portainer.Endpoint, endpointService dataservices.En
 }
 
 func InitialStorageDetection(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) {
+	if endpoint.Kubernetes.Flags.IsServerStorageDetected {
+		return
+	}
+	defer func() {
+		endpoint.Kubernetes.Flags.IsServerStorageDetected = true
+		endpointService.UpdateEndpoint(
+			portainer.EndpointID(endpoint.ID),
+			endpoint,
+		)
+	}()
 	log.Info().Msg("attempting to detect storage classes in the cluster")
 	err := storageDetect(endpoint, endpointService, factory)
 	if err == nil {
@@ -172,3 +197,39 @@ func InitialStorageDetection(endpoint *portainer.Endpoint, endpointService datas
 		log.Err(err).Msg("final error while detecting storage classes")
 	}()
 }
+
+func UpdateEdgeEndpointHeartbeat(endpoint *portainer.Endpoint, settings *portainer.Settings) {
+	if IsEdgeEndpoint(endpoint) {
+		checkInInterval := getEndpointCheckinInterval(endpoint, settings)
+		endpoint.Heartbeat = endpoint.QueryDate-endpoint.LastCheckInDate <= int64(checkInInterval*2+20)
+	}
+}
+
+func getEndpointCheckinInterval(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
+	if endpoint.Edge.AsyncMode {
+		defaultInterval := 60
+		intervals := [][]int{
+			{endpoint.Edge.PingInterval, settings.Edge.PingInterval},
+			{endpoint.Edge.CommandInterval, settings.Edge.CommandInterval},
+			{endpoint.Edge.SnapshotInterval, settings.Edge.SnapshotInterval},
+		}
+
+		for i := 0; i < len(intervals); i++ {
+			effectiveInterval := intervals[i][0]
+			if effectiveInterval <= 0 {
+				effectiveInterval = intervals[i][1]
+			}
+			if effectiveInterval > 0 && effectiveInterval < defaultInterval {
+				defaultInterval = effectiveInterval
+			}
+		}
+
+		return defaultInterval
+	}
+
+	if endpoint.EdgeCheckinInterval > 0 {
+		return endpoint.EdgeCheckinInterval
+	}
+
+	return settings.EdgeAgentCheckinInterval
+}

api/internal/testhelpers/git_service.go

@@ -1,12 +1,32 @@
 package testhelpers
 
-type gitService struct{}
+import portainer "github.com/portainer/portainer/api"
+
+type gitService struct {
+	cloneErr error
+	id       string
+}
 
 // NewGitService creates new mock for portainer.GitService.
-func NewGitService() *gitService {
-	return &gitService{}
+func NewGitService(cloneErr error, id string) portainer.GitService {
+	return &gitService{
+		cloneErr: cloneErr,
+		id:       id,
+	}
 }
 
-func (service *gitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string) error {
-	return nil
+func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error {
+	return g.cloneErr
+}
+
+func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
+	return g.id, nil
+}
+
+func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
+	return nil, nil
+}
+
+func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
+	return nil, nil
 }

api/kubernetes/cli/client.go

@@ -12,12 +12,16 @@ import (
 	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+const (
+	DefaultKubeClientQPS   = 30
+	DefaultKubeClientBurst = 100
+)
+
 type (
 	// ClientFactory is used to create Kubernetes clients
 	ClientFactory struct {
@@ -114,6 +118,9 @@ func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, k
 		return nil, err
 	}
 
+	cliConfig.QPS = DefaultKubeClientQPS
+	cliConfig.Burst = DefaultKubeClientBurst
+
 	cli, err := kubernetes.NewForConfig(cliConfig)
 	if err != nil {
 		return nil, err
@@ -199,7 +206,10 @@ func (factory *ClientFactory) createRemoteClient(endpointURL string) (*kubernete
 	if err != nil {
 		return nil, err
 	}
+
 	config.Insecure = true
+	config.QPS = DefaultKubeClientQPS
+	config.Burst = DefaultKubeClientBurst
 
 	config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
 		return &agentHeaderRoundTripper{
@@ -218,30 +228,13 @@ func buildLocalClient() (*kubernetes.Clientset, error) {
 		return nil, err
 	}
 
+	config.QPS = DefaultKubeClientQPS
+	config.Burst = DefaultKubeClientBurst
+
 	return kubernetes.NewForConfig(config)
 }
 
-func (factory *ClientFactory) PostInitMigrateIngresses() error {
-	endpoints, err := factory.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-	for i := range endpoints {
-		// Early exit if we do not need to migrate!
-		if endpoints[i].PostInitMigrations.MigrateIngresses == false {
-			return nil
-		}
-
-		err := factory.migrateEndpointIngresses(&endpoints[i])
-		if err != nil {
-			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
-		}
-	}
-
-	return nil
-}
-
-func (factory *ClientFactory) migrateEndpointIngresses(e *portainer.Endpoint) error {
+func (factory *ClientFactory) MigrateEndpointIngresses(e *portainer.Endpoint) error {
 	// classes is a list of controllers which have been manually added to the
 	// cluster setup view. These need to all be allowed globally, but then
 	// blocked in specific namespaces which they were not previously allowed in.

api/kubernetes/cli/ingress.go

@@ -102,6 +102,8 @@ func (kcl *KubeClient) GetIngresses(namespace string) ([]models.K8sIngressInfo,
 		}
 		info.Type = classes[info.ClassName]
 		info.Annotations = ingress.Annotations
+		info.Labels = ingress.Labels
+		info.CreationDate = ingress.CreationTimestamp.Time
 
 		// Gather TLS information.
 		for _, v := range ingress.Spec.TLS {

api/kubernetes/cli/service.go

@@ -6,11 +6,12 @@ import (
 	models "github.com/portainer/portainer/api/http/models/kubernetes"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 // GetServices gets all the services for a given namespace in a k8s endpoint.
-func (kcl *KubeClient) GetServices(namespace string) ([]models.K8sServiceInfo, error) {
+func (kcl *KubeClient) GetServices(namespace string, lookupApplications bool) ([]models.K8sServiceInfo, error) {
 	client := kcl.cli.CoreV1().Services(namespace)
 
 	services, err := client.List(context.Background(), metav1.ListOptions{})
@@ -28,7 +29,7 @@ func (kcl *KubeClient) GetServices(namespace string) ([]models.K8sServiceInfo, e
 				NodePort:   int(port.NodePort),
 				Port:       int(port.Port),
 				Protocol:   string(port.Protocol),
-				TargetPort: port.TargetPort.IntValue(),
+				TargetPort: port.TargetPort.String(),
 			})
 		}
 
@@ -40,6 +41,11 @@ func (kcl *KubeClient) GetServices(namespace string) ([]models.K8sServiceInfo, e
 			})
 		}
 
+		var applications []models.K8sApplication
+		if lookupApplications {
+			applications, _ = kcl.getOwningApplication(namespace, service.Spec.Selector)
+		}
+
 		result = append(result, models.K8sServiceInfo{
 			Name:                          service.Name,
 			UID:                           string(service.GetUID()),
@@ -51,6 +57,10 @@ func (kcl *KubeClient) GetServices(namespace string) ([]models.K8sServiceInfo, e
 			IngressStatus:                 ingressStatus,
 			Labels:                        service.GetLabels(),
 			Annotations:                   service.GetAnnotations(),
+			ClusterIPs:                    service.Spec.ClusterIPs,
+			ExternalName:                  service.Spec.ExternalName,
+			ExternalIPs:                   service.Spec.ExternalIPs,
+			Applications:                  applications,
 		})
 	}
 
@@ -77,7 +87,7 @@ func (kcl *KubeClient) CreateService(namespace string, info models.K8sServiceInf
 		port.NodePort = int32(p.NodePort)
 		port.Port = int32(p.Port)
 		port.Protocol = v1.Protocol(p.Protocol)
-		port.TargetPort = intstr.FromInt(p.TargetPort)
+		port.TargetPort = intstr.FromString(p.TargetPort)
 		service.Spec.Ports = append(service.Spec.Ports, port)
 	}
 
@@ -133,7 +143,7 @@ func (kcl *KubeClient) UpdateService(namespace string, info models.K8sServiceInf
 		port.NodePort = int32(p.NodePort)
 		port.Port = int32(p.Port)
 		port.Protocol = v1.Protocol(p.Protocol)
-		port.TargetPort = intstr.FromInt(p.TargetPort)
+		port.TargetPort = intstr.FromString(p.TargetPort)
 		service.Spec.Ports = append(service.Spec.Ports, port)
 	}
 
@@ -151,3 +161,54 @@ func (kcl *KubeClient) UpdateService(namespace string, info models.K8sServiceInf
 	_, err := ServiceClient.Update(context.Background(), &service, metav1.UpdateOptions{})
 	return err
 }
+
+// getOwningApplication gets the application that owns the given service selector.
+func (kcl *KubeClient) getOwningApplication(namespace string, selector map[string]string) ([]models.K8sApplication, error) {
+	if len(selector) == 0 {
+		return nil, nil
+	}
+
+	selectorLabels := labels.SelectorFromSet(selector).String()
+
+	// look for replicasets first, limit 1 (we only support one owner)
+	replicasets, err := kcl.cli.AppsV1().ReplicaSets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selectorLabels, Limit: 1})
+	if err != nil {
+		return nil, err
+	}
+
+	var meta metav1.Object
+	if replicasets != nil && len(replicasets.Items) > 0 {
+		meta = replicasets.Items[0].GetObjectMeta()
+	} else {
+		// otherwise look for matching pods, limit 1 (we only support one owner)
+		pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selectorLabels, Limit: 1})
+		if err != nil {
+			return nil, err
+		}
+
+		if pods == nil || len(pods.Items) == 0 {
+			return nil, nil
+		}
+
+		meta = pods.Items[0].GetObjectMeta()
+	}
+
+	return makeApplication(meta), nil
+}
+
+func makeApplication(meta metav1.Object) []models.K8sApplication {
+	ownerReferences := meta.GetOwnerReferences()
+	if len(ownerReferences) == 0 {
+		return nil
+	}
+
+	// Currently, we only support one owner reference
+	ownerReference := ownerReferences[0]
+	return []models.K8sApplication{
+		{
+			// Only the name is used right now, but we can add more fields in the future
+			Name: ownerReference.Name,
+		},
+	}
+
+}

api/portainer.go

@@ -32,6 +32,20 @@ type (
 	// Authorizations represents a set of authorizations associated to a role
 	Authorizations map[Authorization]bool
 
+	//AutoUpdateSettings represents the git auto sync config for stack deployment
+	AutoUpdateSettings struct {
+		// Auto update interval
+		Interval string `example:"1m30s"`
+		// A UUID generated from client
+		Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"`
+		// Autoupdate job id
+		JobID string `example:"15"`
+		// Force update ignores repo changes
+		ForceUpdate bool `example:"false"`
+		// Pull latest image
+		ForcePullImage bool `example:"false"`
+	}
+
 	// AzureCredentials represents the credentials used to connect to an Azure
 	// environment(endpoint).
 	AzureCredentials struct {
@@ -207,15 +221,21 @@ type (
 		GpuUseList              []string          `json:"GpuUseList"`
 	}
 
-	// DockerSnapshotRaw represents all the information related to a snapshot as returned by the Docker API
+	// DockerContainerSnapshot is an extent of Docker's Container struct
+	// It contains some information of Docker's ContainerJSON struct
+	DockerContainerSnapshot struct {
+		types.Container
+		Env []string `json:"Env,omitempty"` // EE-5240
+	}
 
+	// DockerSnapshotRaw represents all the information related to a snapshot as returned by the Docker API
 	DockerSnapshotRaw struct {
-		Containers []types.Container       `json:"Containers" swaggerignore:"true"`
-		Volumes    volume.VolumeListOKBody `json:"Volumes" swaggerignore:"true"`
-		Networks   []types.NetworkResource `json:"Networks" swaggerignore:"true"`
-		Images     []types.ImageSummary    `json:"Images" swaggerignore:"true"`
-		Info       types.Info              `json:"Info" swaggerignore:"true"`
-		Version    types.Version           `json:"Version" swaggerignore:"true"`
+		Containers []DockerContainerSnapshot `json:"Containers" swaggerignore:"true"`
+		Volumes    volume.VolumeListOKBody   `json:"Volumes" swaggerignore:"true"`
+		Networks   []types.NetworkResource   `json:"Networks" swaggerignore:"true"`
+		Images     []types.ImageSummary      `json:"Images" swaggerignore:"true"`
+		Info       types.Info                `json:"Info" swaggerignore:"true"`
+		Version    types.Version             `json:"Version" swaggerignore:"true"`
 	}
 
 	// EdgeGroup represents an Edge group
@@ -368,6 +388,9 @@ type (
 		LastCheckInDate int64
 		// QueryDate of each query with the endpoints list
 		QueryDate int64
+		// Heartbeat indicates the heartbeat status of an edge environment
+		Heartbeat bool `json:"Heartbeat" example:"true"`
+
 		// IsEdgeDevice marks if the environment was created as an EdgeDevice
 		IsEdgeDevice bool
 		// Whether the device has been trusted or not by the user
@@ -376,21 +399,14 @@ type (
 		// Whether we need to run any "post init migrations".
 		PostInitMigrations EndpointPostInitMigrations `json:"PostInitMigrations"`
 
-		Edge struct {
-			// Whether the device has been started in edge async mode
-			AsyncMode bool
-			// The ping interval for edge agent - used in edge async mode [seconds]
-			PingInterval int `json:"PingInterval" example:"60"`
-			// The snapshot interval for edge agent - used in edge async mode [seconds]
-			SnapshotInterval int `json:"SnapshotInterval" example:"60"`
-			// The command list interval for edge agent - used in edge async mode [seconds]
-			CommandInterval int `json:"CommandInterval" example:"60"`
-		}
+		Edge EnvironmentEdgeSettings
 
 		Agent struct {
 			Version string `example:"1.0.0"`
 		}
 
+		EnableGPUManagement bool `json:"EnableGPUManagement"`
+
 		// Deprecated fields
 		// Deprecated in DBVersion == 4
 		TLS           bool   `json:"TLS,omitempty"`
@@ -406,6 +422,17 @@ type (
 		Tags []string `json:"Tags"`
 	}
 
+	EnvironmentEdgeSettings struct {
+		// Whether the device has been started in edge async mode
+		AsyncMode bool
+		// The ping interval for edge agent - used in edge async mode [seconds]
+		PingInterval int `json:"PingInterval" example:"60"`
+		// The snapshot interval for edge agent - used in edge async mode [seconds]
+		SnapshotInterval int `json:"SnapshotInterval" example:"60"`
+		// The command list interval for edge agent - used in edge async mode [seconds]
+		CommandInterval int `json:"CommandInterval" example:"60"`
+	}
+
 	// EndpointAuthorizations represents the authorizations associated to a set of environments(endpoints)
 	EndpointAuthorizations map[EndpointID]Authorizations
 
@@ -480,6 +507,7 @@ type (
 	// EndpointPostInitMigrations
 	EndpointPostInitMigrations struct {
 		MigrateIngresses bool `json:"MigrateIngresses"`
+		MigrateGPUs      bool `json:"MigrateGPUs"`
 	}
 
 	// Extension represents a deprecated Portainer extension
@@ -563,9 +591,12 @@ type (
 		Flags         KubernetesFlags         `json:"Flags"`
 	}
 
+	// KubernetesFlags are used to detect if we need to run initial cluster
+	// detection again.
 	KubernetesFlags struct {
-		IsServerMetricsDetected bool `json:"IsServerMetricsDetected"`
-		IsServerStorageDetected bool `json:"IsServerStorageDetected"`
+		IsServerMetricsDetected      bool `json:"IsServerMetricsDetected"`
+		IsServerIngressClassDetected bool `json:"IsServerIngressClassDetected"`
+		IsServerStorageDetected      bool `json:"IsServerStorageDetected"`
 	}
 
 	// KubernetesSnapshot represents a snapshot of a specific Kubernetes environment(endpoint) at a specific time
@@ -911,7 +942,8 @@ type (
 			PingInterval int `json:"PingInterval" example:"5"`
 			// The snapshot interval for edge agent - used in edge async mode (in seconds)
 			SnapshotInterval int `json:"SnapshotInterval" example:"5"`
-			// EdgeAsyncMode enables edge async mode by default
+
+			// Deprecated 2.18
 			AsyncMode bool
 		}
 
@@ -977,7 +1009,7 @@ type (
 		// Only applies when deploying stack with multiple files
 		AdditionalFiles []string `json:"AdditionalFiles"`
 		// The auto update settings of a git stack
-		AutoUpdate *StackAutoUpdate `json:"AutoUpdate"`
+		AutoUpdate *AutoUpdateSettings `json:"AutoUpdate"`
 		// The stack deployment option
 		Option *StackOption `json:"Option"`
 		// The git config of this stack
@@ -990,16 +1022,6 @@ type (
 		IsComposeFormat bool `example:"false"`
 	}
 
-	//StackAutoUpdate represents the git auto sync config for stack deployment
-	StackAutoUpdate struct {
-		// Auto update interval
-		Interval string `example:"1m30s"`
-		// A UUID generated from client
-		Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"`
-		// Autoupdate job id
-		JobID string `example:"15"`
-	}
-
 	// StackOption represents the options for stack deployment
 	StackOption struct {
 		// Prune services that are no longer referenced
@@ -1270,8 +1292,6 @@ type (
 	UserThemeSettings struct {
 		// Color represents the color theme of the UI
 		Color string `json:"color" example:"dark" enums:"dark,light,highcontrast,auto"`
-		// SubtleUpgradeButton indicates if the upgrade banner should be displayed in a subtle way
-		SubtleUpgradeButton bool `json:"subtleUpgradeButton"`
 	}
 
 	// Webhook represents a url webhook that can be used to update a service
@@ -1376,10 +1396,10 @@ type (
 
 	// GitService represents a service for managing Git
 	GitService interface {
-		CloneRepository(destination string, repositoryURL, referenceName, username, password string) error
-		LatestCommitID(repositoryURL, referenceName, username, password string) (string, error)
-		ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error)
-		ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string) ([]string, error)
+		CloneRepository(destination string, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error
+		LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error)
+		ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error)
+		ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error)
 	}
 
 	// OpenAMTService represents a service for managing OpenAMT
@@ -1416,7 +1436,7 @@ type (
 		DeleteIngresses(reqs models.K8sIngressDeleteRequests) error
 		CreateService(namespace string, service models.K8sServiceInfo) error
 		UpdateService(namespace string, service models.K8sServiceInfo) error
-		GetServices(namespace string) ([]models.K8sServiceInfo, error)
+		GetServices(namespace string, lookupApplications bool) ([]models.K8sServiceInfo, error)
 		DeleteServices(reqs models.K8sServiceDeleteRequests) error
 		GetNodesLimits() (K8sNodesLimits, error)
 		GetNamespaceAccessPolicies() (map[string]K8sNamespaceAccessPolicy, error)
@@ -1494,7 +1514,7 @@ type (
 
 const (
 	// APIVersion is the version number of the Portainer API
-	APIVersion = "2.18.0"
+	APIVersion = "2.18.2"
 	// Edition is what this edition of Portainer is called
 	Edition = PortainerCE
 	// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax
@@ -1541,7 +1561,9 @@ const (
 )
 
 // List of supported features
-var SupportedFeatureFlags = []featureflags.Feature{}
+var SupportedFeatureFlags = []featureflags.Feature{
+	"fdo",
+}
 
 const (
 	_ AuthenticationMethod = iota

api/stacks/deployments/deploy.go

@@ -2,11 +2,11 @@ package deployments
 
 import (
 	"fmt"
-	"strings"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 
 	"github.com/pkg/errors"
@@ -36,6 +36,11 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 		return nil // do nothing if it isn't a git-based stack
 	}
 
+	endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to find the environment %v associated to the stack %v", stack.EndpointID, stack.ID)
+	}
+
 	author := stack.UpdatedBy
 	if author == "" {
 		author = stack.CreatedBy
@@ -48,44 +53,27 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 			Str("author", author).
 			Str("stack", stack.Name).
 			Int("endpoint_id", int(stack.EndpointID)).
-			Msg("cannot autoupdate a stack, stack author user is missing")
+			Msg("cannot auto update a stack, stack author user is missing")
 
 		return &StackAuthorMissingErr{int(stack.ID), author}
 	}
 
-	username, password := "", ""
-	if stack.GitConfig.Authentication != nil {
-		username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password
-	}
+	var gitCommitChangedOrForceUpdate bool
+	if !stack.FromAppTemplate {
+		updated, newHash, err := update.UpdateGitObject(gitService, fmt.Sprintf("stack:%d", stackID), stack.GitConfig, false, stack.ProjectPath)
+		if err != nil {
+			return err
+		}
 
-	newHash, err := gitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, username, password)
-	if err != nil {
-		return errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID)
-	}
-
-	if strings.EqualFold(newHash, string(stack.GitConfig.ConfigHash)) {
-		return nil
-	}
-
-	cloneParams := &cloneRepositoryParameters{
-		url:   stack.GitConfig.URL,
-		ref:   stack.GitConfig.ReferenceName,
-		toDir: stack.ProjectPath,
-	}
-	if stack.GitConfig.Authentication != nil {
-		cloneParams.auth = &gitAuth{
-			username: username,
-			password: password,
+		if updated {
+			stack.GitConfig.ConfigHash = newHash
+			stack.UpdateDate = time.Now().Unix()
+			gitCommitChangedOrForceUpdate = updated
 		}
 	}
 
-	if err := cloneGitRepository(gitService, cloneParams); err != nil {
-		return errors.WithMessagef(err, "failed to do a fresh clone of the stack %v", stack.ID)
-	}
-
-	endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID)
-	if err != nil {
-		return errors.WithMessagef(err, "failed to find the environment %v associated to the stack %v", stack.EndpointID, stack.ID)
+	if !gitCommitChangedOrForceUpdate {
+		return nil
 	}
 
 	registries, err := getUserRegistries(datastore, user, endpoint.ID)
@@ -111,14 +99,12 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 
 		err := deployer.DeployKubernetesStack(stack, endpoint, user)
 		if err != nil {
-			return errors.WithMessagef(err, "failed to deploy a kubternetes app stack %v", stackID)
+			return errors.WithMessagef(err, "failed to deploy a kubernetes app stack %v", stackID)
 		}
 	default:
 		return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type)
 	}
 
-	stack.UpdateDate = time.Now().Unix()
-	stack.GitConfig.ConfigHash = newHash
 	if err := datastore.Stack().UpdateStack(stack.ID, stack); err != nil {
 		return errors.WithMessagef(err, "failed to update the stack %v", stack.ID)
 	}
@@ -150,22 +136,3 @@ func getUserRegistries(datastore dataservices.DataStore, user *portainer.User, e
 
 	return filteredRegistries, nil
 }
-
-type cloneRepositoryParameters struct {
-	url   string
-	ref   string
-	toDir string
-	auth  *gitAuth
-}
-
-type gitAuth struct {
-	username string
-	password string
-}
-
-func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error {
-	if cloneParams.auth != nil {
-		return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password)
-	}
-	return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "")
-}

api/stacks/deployments/deploy_test.go

@@ -6,33 +6,13 @@ import (
 	"testing"
 
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 
 	portainer "github.com/portainer/portainer/api"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
-type gitService struct {
-	cloneErr error
-	id       string
-}
-
-func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
-	return g.cloneErr
-}
-
-func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	return g.id, nil
-}
-
-func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
-	return nil, nil
-}
-
-func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
-	return nil, nil
-}
-
 type noopDeployer struct{}
 
 func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {
@@ -67,7 +47,7 @@ func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) {
 	err = store.Stack().Create(&portainer.Stack{ID: 1, CreatedBy: "admin"})
 	assert.NoError(t, err, "failed to create a test stack")
 
-	err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""})
+	err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(nil, ""))
 	assert.NoError(t, err)
 }
 
@@ -81,6 +61,11 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) {
 	err := store.User().Create(admin)
 	assert.NoError(t, err, "error creating an admin")
 
+	err = store.Endpoint().Create(&portainer.Endpoint{
+		ID: 0,
+	})
+	assert.NoError(t, err, "error creating environment")
+
 	err = store.Stack().Create(&portainer.Stack{
 		ID:          1,
 		CreatedBy:   "admin",
@@ -92,7 +77,7 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) {
 		}})
 	assert.NoError(t, err, "failed to create a test stack")
 
-	err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"})
+	err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(nil, "oldHash"))
 	assert.NoError(t, err)
 }
 
@@ -105,6 +90,11 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
 	err := store.User().Create(admin)
 	assert.NoError(t, err, "error creating an admin")
 
+	err = store.Endpoint().Create(&portainer.Endpoint{
+		ID: 0,
+	})
+	assert.NoError(t, err, "error creating environment")
+
 	err = store.Stack().Create(&portainer.Stack{
 		ID:        1,
 		CreatedBy: "admin",
@@ -115,7 +105,7 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
 		}})
 	assert.NoError(t, err, "failed to create a test stack")
 
-	err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"})
+	err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(cloneErr, "newHash"))
 	assert.Error(t, err)
 	assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup")
 }
@@ -142,7 +132,9 @@ func Test_redeployWhenChanged(t *testing.T) {
 			URL:           "url",
 			ReferenceName: "ref",
 			ConfigHash:    "oldHash",
-		}}
+		},
+	}
+
 	err = store.Stack().Create(&stack)
 	assert.NoError(t, err, "failed to create a test stack")
 
@@ -150,7 +142,7 @@ func Test_redeployWhenChanged(t *testing.T) {
 		stack.Type = portainer.DockerComposeStack
 		store.Stack().UpdateStack(stack.ID, &stack)
 
-		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
 		assert.NoError(t, err)
 	})
 
@@ -158,7 +150,7 @@ func Test_redeployWhenChanged(t *testing.T) {
 		stack.Type = portainer.DockerSwarmStack
 		store.Stack().UpdateStack(stack.ID, &stack)
 
-		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
 		assert.NoError(t, err)
 	})
 
@@ -166,7 +158,7 @@ func Test_redeployWhenChanged(t *testing.T) {
 		stack.Type = portainer.KubernetesStack
 		store.Stack().UpdateStack(stack.ID, &stack)
 
-		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
 		assert.NoError(t, err)
 	})
 }

api/stacks/stackbuilders/stack_git_builder.go

@@ -67,6 +67,8 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho
 
 	repoConfig.URL = payload.URL
 	repoConfig.ReferenceName = payload.ReferenceName
+	repoConfig.TLSSkipVerify = payload.TLSSkipVerify
+
 	repoConfig.ConfigFilePath = payload.ComposeFile
 	if payload.ComposeFile == "" {
 		repoConfig.ConfigFilePath = filesystem.ComposeFileDefaultName

api/stacks/stackbuilders/stack_payload.go

@@ -18,7 +18,7 @@ type StackPayload struct {
 	// A list of environment(endpoint) variables used during stack deployment
 	Env []portainer.Pair
 	// Optional auto update configuration
-	AutoUpdate *portainer.StackAutoUpdate
+	AutoUpdate *portainer.AutoUpdateSettings
 	// Whether the stack is from a app template
 	FromAppTemplate bool `example:"false"`
 	// Kubernetes stack name
@@ -52,4 +52,6 @@ type RepositoryConfigPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true
 	// and RepositoryGitCredentialID is 0
 	Password string `example:"myGitPassword"`
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
 }